From f77d4c494aa1adb34e7631685aa48c8ddbfc9134 Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Fri, 9 Feb 2024 14:32:06 -0800 Subject: [PATCH 01/25] go: sqle/cluster: When in standby mode, take the epoch of the primary. When in detected_broken_config, become standby if we see a higher numbered primary. --- .../doltcore/sqle/cluster/interceptors.go | 26 ++-- .../tests/sql-server-cluster.yaml | 111 ++++++++++++++++++ 2 files changed, 126 insertions(+), 11 deletions(-) diff --git a/go/libraries/doltcore/sqle/cluster/interceptors.go b/go/libraries/doltcore/sqle/cluster/interceptors.go index 049be9e3fd9..c078aa4bcc6 100644 --- a/go/libraries/doltcore/sqle/cluster/interceptors.go +++ b/go/libraries/doltcore/sqle/cluster/interceptors.go @@ -204,11 +204,11 @@ type serverinterceptor struct { func (si *serverinterceptor) Stream() grpc.StreamServerInterceptor { return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - fromStandby := false + fromClusterMember := false if md, ok := metadata.FromIncomingContext(ss.Context()); ok { - fromStandby = si.handleRequestHeaders(md) + fromClusterMember = si.handleRequestHeaders(md) } - if fromStandby { + if fromClusterMember { if err := si.authenticate(ss.Context()); err != nil { return err } @@ -236,11 +236,11 @@ func (si *serverinterceptor) Stream() grpc.StreamServerInterceptor { func (si *serverinterceptor) Unary() grpc.UnaryServerInterceptor { return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - fromStandby := false + fromClusterMember := false if md, ok := metadata.FromIncomingContext(ctx); ok { - fromStandby = si.handleRequestHeaders(md) + fromClusterMember = si.handleRequestHeaders(md) } - if fromStandby { + if fromClusterMember { if err := si.authenticate(ctx); err != nil { return nil, err } @@ -271,9 +271,9 @@ func (si *serverinterceptor) handleRequestHeaders(header metadata.MD) bool { epochs := header.Get(clusterRoleEpochHeader) roles := header.Get(clusterRoleHeader) if len(epochs) > 0 && len(roles) > 0 { - if roles[0] == string(RolePrimary) && role == RolePrimary { + if roles[0] == string(RolePrimary) { if reqepoch, err := strconv.Atoi(epochs[0]); err == nil { - if reqepoch == epoch { + if reqepoch == epoch && role == RolePrimary { // Misconfiguration in the cluster means this // server and its standby are marked as Primary // at the same epoch. We will become standby @@ -282,13 +282,17 @@ func (si *serverinterceptor) handleRequestHeaders(header metadata.MD) bool { si.lgr.Errorf("cluster: serverinterceptor: this server and its standby replica are both primary at the same epoch. force transitioning to detected_broken_config.") si.roleSetter(string(RoleDetectedBrokenConfig), reqepoch) } else if reqepoch > epoch { - // The client replicating to us thinks it is the primary at a higher epoch than us. - si.lgr.Warnf("cluster: serverinterceptor: this server is primary at epoch %d. the server replicating to it is primary at epoch %d. force transitioning to standby.", epoch, reqepoch) + if role == RolePrimary { + // The client replicating to us thinks it is the primary at a higher epoch than us. + si.lgr.Warnf("cluster: serverinterceptor: this server is primary at epoch %d. the server replicating to it is primary at epoch %d. force transitioning to standby.", epoch, reqepoch) + } else if role == RoleDetectedBrokenConfig { + si.lgr.Warnf("cluster: serverinterceptor: this server is detected_broken_config at epoch %d. the server replicating to it is primary at epoch %d. transitioning to standby.", epoch, reqepoch) + } si.roleSetter(string(RoleStandby), reqepoch) } } } - // returns true if the request was from a standby replica, false otherwise + // returns true if the request was from a cluster replica, false otherwise return true } return false diff --git a/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml b/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml index 0d62bd39bf2..739b6d40464 100644 --- a/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml +++ b/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml @@ -337,6 +337,117 @@ tests: queries: - exec: "use repo1" - exec: "create table vals (i int primary key)" +- name: standby server takes primary epoch + multi_repos: + - name: server1 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:3852/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:3852/repo2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:3852/{database} + bootstrap_role: primary + bootstrap_epoch: 10 + remotesapi: + port: 3851 + server: + args: ["--port", "3309"] + port: 3309 + - name: server2 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:3851/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:3851/repo2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:3851/{database} + bootstrap_role: standby + bootstrap_epoch: 1 + remotesapi: + port: 3852 + server: + args: ["--config", "server.yaml"] + port: 3310 + connections: + - on: server1 + queries: + - exec: "use repo1" + - exec: "create table vals (i int primary key)" + - exec: "insert into vals values (1),(2),(3),(4),(5)" + restart_server: + args: ["--config", "server.yaml"] + - on: server1 + queries: + - exec: "use dolt_cluster" + - query: "select `database`, standby_remote, role, epoch, replication_lag_millis, current_error from dolt_cluster_status order by `database` asc" + result: + columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] + rows: + - ["repo1","standby","primary","10","0","NULL"] + - ["repo2","standby","primary","10","0","NULL"] + retry_attempts: 100 + - on: server2 + queries: + - exec: "use repo1" + - query: "select count(*) from vals" + result: + columns: ["count(*)"] + rows: [["5"]] + - query: "select @@global.dolt_cluster_role_epoch" + result: + columns: ["@@global.dolt_cluster_role_epoch"] + rows: [["10"]] + - on: server1 + queries: + - exec: "use repo1" + - exec: "call dolt_assume_cluster_role('primary', 11)" + - exec: "insert into vals values (6),(7),(8),(9),(10)" + - exec: "use dolt_cluster" + - query: "select `database`, standby_remote, role, epoch, replication_lag_millis, current_error from dolt_cluster_status order by `database` asc" + result: + columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] + rows: + - ["repo1","standby","primary","11","0","NULL"] + - ["repo2","standby","primary","11","0","NULL"] + retry_attempts: 100 + - on: server2 + queries: + - exec: "use repo1" + - query: "select count(*) from vals" + result: + columns: ["count(*)"] + rows: [["10"]] + - query: "select @@global.dolt_cluster_role_epoch" + result: + columns: ["@@global.dolt_cluster_role_epoch"] + rows: [["11"]] - name: standby transitioned to primary becomes read write multi_repos: - name: server1 From 58d3847aa97189c25b99f205cbdeb87a5be6f28c Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Fri, 9 Feb 2024 15:18:06 -0800 Subject: [PATCH 02/25] /go/performance/utils/sysbench_runner: wip, refactoring to make better --- .../utils/sysbench_runner/benchmark.go | 7 + go/performance/utils/sysbench_runner/dolt.go | 304 +++++------------- .../utils/sysbench_runner/dolt_old.go | 291 +++++++++++++++++ .../utils/sysbench_runner/dolt_server.go | 97 ++++++ go/performance/utils/sysbench_runner/run.go | 9 +- .../utils/sysbench_runner/server.go | 8 + .../utils/sysbench_runner/sysbench.go | 192 +++++++++++ 7 files changed, 686 insertions(+), 222 deletions(-) create mode 100644 go/performance/utils/sysbench_runner/benchmark.go create mode 100644 go/performance/utils/sysbench_runner/dolt_old.go create mode 100644 go/performance/utils/sysbench_runner/dolt_server.go create mode 100644 go/performance/utils/sysbench_runner/server.go create mode 100644 go/performance/utils/sysbench_runner/sysbench.go diff --git a/go/performance/utils/sysbench_runner/benchmark.go b/go/performance/utils/sysbench_runner/benchmark.go new file mode 100644 index 00000000000..b7c272771a9 --- /dev/null +++ b/go/performance/utils/sysbench_runner/benchmark.go @@ -0,0 +1,7 @@ +package sysbench_runner + +import "context" + +type Benchmarker interface { + Benchmark(ctx context.Context) (Results, error) +} diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go index 3bae67be6b8..1ddcce517f4 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -1,170 +1,100 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package sysbench_runner import ( "context" - "fmt" + "github.com/dolthub/dolt/go/store/types" "os" - "os/exec" - "os/signal" "path/filepath" - "sync" "syscall" - "time" - - "github.com/dolthub/dolt/go/store/types" - - "golang.org/x/sync/errgroup" ) const ( - dbName = "test" - luaPath = "?.lua" - bigEmptyRepo = "max-hoffman/big-empty" - nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" + doltConfigUsernameKey = "user.name" + doltConfigEmailKey = "user.email" + doltBenchmarkUser = "benchmark" + doltBenchmarkEmail = "benchmark@dolthub.com" + doltConfigCommand = "config" + doltConfigGlobalFlag = "--global" + doltConfigGetFlag = "--get" + doltConfigAddFlag = "--add" + doltCloneCommand = "clone" + doltVersionCommand = "version" + doltInitCommand = "init" + dbName = "test" + bigEmptyRepo = "max-hoffman/big-empty" + nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" ) -var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } - -// BenchmarkDolt benchmarks dolt based on the provided configurations -func BenchmarkDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return nil, err - } - - err = DoltVersion(ctx, serverConfig.ServerExec) - if err != nil { - return nil, err - } +type doltBenchmarkerImpl struct { + dir string // cwd + config *Config + serverConfig *ServerConfig +} - err = UpdateDoltConfig(ctx, serverConfig.ServerExec) - if err != nil { - return nil, err - } +var _ Benchmarker = &doltBenchmarkerImpl{} - testRepo, err := initDoltRepo(ctx, serverConfig, config.NomsBinFormat) - if err != nil { - return nil, err +func NewDoltBenchmarker(dir string, config *Config, serverConfig *ServerConfig) *doltBenchmarkerImpl { + return &doltBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, } +} - withKeyCtx, cancel := context.WithCancel(ctx) - gServer, serverCtx := errgroup.WithContext(withKeyCtx) - - server := getServer(serverCtx, serverConfig, testRepo, serverParams) - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - // launch the dolt server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(5 * time.Second) - - tests, err := GetTests(config, serverConfig, nil) +// checkSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary +func (b *doltBenchmarkerImpl) checkSetDoltConfig(ctx context.Context, serverExec, key, val string) error { + check := ExecCommand(ctx, b.serverConfig.ServerExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigGetFlag, key) + err := check.Run() if err != nil { - return nil, err - } - - results := make(Results, 0) - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - results = append(results, r) + // config get calls exit with 1 if not set + if err.Error() != "exit status 1" { + return err } - } - - // send signal to dolt server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - fmt.Println(err) - close(quit) - wg.Wait() - return nil, err + set := ExecCommand(ctx, serverExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigAddFlag, key, val) + err := set.Run() + if err != nil { + return err } } + return nil +} - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - err = os.RemoveAll(testRepo) +func (b *doltBenchmarkerImpl) updateGlobalConfig(ctx context.Context) error { + err := checkSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigUsernameKey, doltBenchmarkUser) if err != nil { - return nil, err + return err } - - return results, nil + return checkSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigEmailKey, doltBenchmarkEmail) } -// DoltVersion ensures the dolt binary can run -func DoltVersion(ctx context.Context, serverExec string) error { - doltVersion := ExecCommand(ctx, serverExec, "version") - return doltVersion.Run() +func (b *doltBenchmarkerImpl) checkInstallation(ctx context.Context) error { + version := ExecCommand(ctx, b.serverConfig.ServerExec, doltVersionCommand) + return version.Run() } // initDoltRepo initializes a dolt repo and returns the repo path -func initDoltRepo(ctx context.Context, config *ServerConfig, nbf string) (string, error) { - cwd, err := os.Getwd() - if err != nil { - return "", err - } - - testRepo := filepath.Join(cwd, dbName) - if nbf == types.Format_LD_1.VersionString() { - err := ExecCommand(ctx, config.ServerExec, "clone", bigEmptyRepo, dbName).Run() +func (b *doltBenchmarkerImpl) initDoltRepo(ctx context.Context) (string, error) { + testRepo := filepath.Join(b.dir, dbName) + if b.config.NomsBinFormat == types.Format_LD_1.VersionString() { + err := ExecCommand(ctx, b.serverConfig.ServerExec, doltCloneCommand, bigEmptyRepo, dbName).Run() if err != nil { return "", err } return testRepo, nil } - err = os.MkdirAll(testRepo, os.ModePerm) + + err := os.MkdirAll(testRepo, os.ModePerm) if err != nil { return "", err } - if nbf != "" { - if err = os.Setenv(nbfEnvVar, nbf); err != nil { + if b.config.NomsBinFormat != "" { + if err = os.Setenv(nbfEnvVar, b.config.NomsBinFormat); err != nil { return "", err } } - doltInit := ExecCommand(ctx, config.ServerExec, "init") + doltInit := ExecCommand(ctx, b.serverConfig.ServerExec, doltInitCommand) doltInit.Dir = testRepo err = doltInit.Run() if err != nil { @@ -174,123 +104,55 @@ func initDoltRepo(ctx context.Context, config *ServerConfig, nbf string) (string return testRepo, nil } -// UpdateDoltConfig updates the dolt config if necessary -func UpdateDoltConfig(ctx context.Context, serverExec string) error { - err := checkSetDoltConfig(ctx, serverExec, "user.name", "benchmark") +func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { + err := b.checkInstallation(ctx) if err != nil { - return err + return nil, err } - return checkSetDoltConfig(ctx, serverExec, "user.email", "benchmark@dolthub.com") -} -// checkSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary -func checkSetDoltConfig(ctx context.Context, serverExec, key, val string) error { - check := ExecCommand(ctx, serverExec, "config", "--global", "--get", key) - err := check.Run() + err = b.updateGlobalConfig(ctx) if err != nil { - // config get calls exit with 1 if not set - if err.Error() != "exit status 1" { - return err - } - - set := ExecCommand(ctx, serverExec, "config", "--global", "--add", key, val) - err := set.Run() - if err != nil { - return err - } - } - - return nil -} - -// getServer returns a exec.Cmd for a dolt server -func getServer(ctx context.Context, config *ServerConfig, testRepo string, params []string) *exec.Cmd { - server := ExecCommand(ctx, config.ServerExec, params...) - server.Dir = testRepo - return server -} - -// sysbenchPrepare returns a exec.Cmd for running the sysbench prepare step -func sysbenchPrepare(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { - cmd := exec.CommandContext(ctx, "sysbench", test.Prepare()...) - if test.FromScript { - lp := filepath.Join(scriptDir, luaPath) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) - } - return cmd -} - -// sysbenchRun returns a exec.Cmd for running the sysbench run step -func sysbenchRun(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { - cmd := exec.CommandContext(ctx, "sysbench", test.Run()...) - if test.FromScript { - lp := filepath.Join(scriptDir, luaPath) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) - } - return cmd -} - -// sysbenchPrepare returns a exec.Cmd for running the sysbench cleanup step -func sysbenchCleanup(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { - cmd := ExecCommand(ctx, "sysbench", test.Cleanup()...) - if test.FromScript { - lp := filepath.Join(scriptDir, luaPath) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) + return nil, err } - return cmd -} - -// benchmark runs a sysbench benchmark against a server calling prepare, run, cleanup -func benchmark( - ctx context.Context, - test *Test, - config *Config, - serverConfig *ServerConfig, - stampFunc func() string, - suiteId string, -) (*Result, error) { - prepare := sysbenchPrepare(ctx, test, config.ScriptDir) - run := sysbenchRun(ctx, test, config.ScriptDir) - cleanup := sysbenchCleanup(ctx, test, config.ScriptDir) - fmt.Println("Running test ", test.Name) - - out, err := prepare.Output() + testRepo, err := b.initDoltRepo(ctx) if err != nil { - fmt.Println(string(out)) return nil, err } - out, err = run.Output() + serverParams, err := b.serverConfig.GetServerArgs() if err != nil { - fmt.Print(string(out)) return nil, err } - if Debug == true { - fmt.Print(string(out)) + server := NewDoltServer(ctx, testRepo, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start(ctx) + if err != nil { + return nil, err } - r, err := FromOutputResult(out, config, serverConfig, test, suiteId, nil) + tests, err := GetTests(b.config, b.serverConfig, nil) if err != nil { return nil, err } - r.Stamp(stampFunc) - - return r, cleanup.Run() -} - -// fromChannelResults collects all Results from the given channel and returns them -func fromChannelResults(rc chan *Result) Results { results := make(Results, 0) - for r := range rc { - if r != nil { + for i := 0; i < b.config.Runs; i++ { + for _, test := range tests { + tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop(ctx) + return nil, err + } results = append(results, r) } } - return results + + err = server.Stop(ctx) + if err != nil { + return nil, err + } + + return results, os.RemoveAll(testRepo) } diff --git a/go/performance/utils/sysbench_runner/dolt_old.go b/go/performance/utils/sysbench_runner/dolt_old.go new file mode 100644 index 00000000000..876bb2f11a9 --- /dev/null +++ b/go/performance/utils/sysbench_runner/dolt_old.go @@ -0,0 +1,291 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/dolthub/dolt/go/store/types" +) + +const ( + //dbName = "test" + luaPath = "?.lua" + //bigEmptyRepo = "max-hoffman/big-empty" + //nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" +) + +var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } + +//// BenchmarkDolt benchmarks dolt based on the provided configurations +//func BenchmarkDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { +// serverParams, err := serverConfig.GetServerArgs() +// if err != nil { +// return nil, err +// } +// +// err = DoltVersion(ctx, serverConfig.ServerExec) +// if err != nil { +// return nil, err +// } +// +// err = UpdateDoltConfig(ctx, serverConfig.ServerExec) +// if err != nil { +// return nil, err +// } +// +// testRepo, err := initDoltRepo(ctx, serverConfig, config.NomsBinFormat) +// if err != nil { +// return nil, err +// } +// +// withKeyCtx, cancel := context.WithCancel(ctx) +// gServer, serverCtx := errgroup.WithContext(withKeyCtx) +// +// server := getServer(serverCtx, serverConfig, testRepo, serverParams) +// +// // handle user interrupt +// quit := make(chan os.Signal, 1) +// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) +// var wg sync.WaitGroup +// wg.Add(1) +// go func() { +// <-quit +// defer wg.Done() +// signal.Stop(quit) +// cancel() +// }() +// +// // launch the dolt server +// gServer.Go(func() error { +// return server.Run() +// }) +// +// // sleep to allow the server to start +// time.Sleep(5 * time.Second) +// +// tests, err := GetTests(config, serverConfig, nil) +// if err != nil { +// return nil, err +// } +// +// results := make(Results, 0) +// for i := 0; i < config.Runs; i++ { +// for _, test := range tests { +// r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) +// if err != nil { +// close(quit) +// wg.Wait() +// return nil, err +// } +// results = append(results, r) +// } +// } +// +// // send signal to dolt server +// quit <- syscall.SIGTERM +// +// err = gServer.Wait() +// if err != nil { +// // we expect a kill error +// // we only exit in error if this is not the +// // error +// if err.Error() != "signal: killed" { +// fmt.Println(err) +// close(quit) +// wg.Wait() +// return nil, err +// } +// } +// +// fmt.Println("Successfully killed server") +// close(quit) +// wg.Wait() +// +// err = os.RemoveAll(testRepo) +// if err != nil { +// return nil, err +// } +// +// return results, nil +//} + +// DoltVersion ensures the dolt binary can run +func DoltVersion(ctx context.Context, serverExec string) error { + doltVersion := ExecCommand(ctx, serverExec, "version") + return doltVersion.Run() +} + +// initDoltRepo initializes a dolt repo and returns the repo path +func initDoltRepo(ctx context.Context, config *ServerConfig, nbf string) (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + + testRepo := filepath.Join(cwd, dbName) + if nbf == types.Format_LD_1.VersionString() { + err := ExecCommand(ctx, config.ServerExec, "clone", bigEmptyRepo, dbName).Run() + if err != nil { + return "", err + } + return testRepo, nil + } + err = os.MkdirAll(testRepo, os.ModePerm) + if err != nil { + return "", err + } + + if nbf != "" { + if err = os.Setenv(nbfEnvVar, nbf); err != nil { + return "", err + } + } + + doltInit := ExecCommand(ctx, config.ServerExec, "init") + doltInit.Dir = testRepo + err = doltInit.Run() + if err != nil { + return "", err + } + + return testRepo, nil +} + +// UpdateDoltConfig updates the dolt config if necessary +func UpdateDoltConfig(ctx context.Context, serverExec string) error { + err := checkSetDoltConfig(ctx, serverExec, "user.name", "benchmark") + if err != nil { + return err + } + return checkSetDoltConfig(ctx, serverExec, "user.email", "benchmark@dolthub.com") +} + +// checkSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary +func checkSetDoltConfig(ctx context.Context, serverExec, key, val string) error { + check := ExecCommand(ctx, serverExec, "config", "--global", "--get", key) + err := check.Run() + if err != nil { + // config get calls exit with 1 if not set + if err.Error() != "exit status 1" { + return err + } + + set := ExecCommand(ctx, serverExec, "config", "--global", "--add", key, val) + err := set.Run() + if err != nil { + return err + } + } + + return nil +} + +// getServer returns a exec.Cmd for a dolt server +func getServer(ctx context.Context, config *ServerConfig, testRepo string, params []string) *exec.Cmd { + server := ExecCommand(ctx, config.ServerExec, params...) + server.Dir = testRepo + return server +} + +// sysbenchPrepare returns a exec.Cmd for running the sysbench prepare step +func sysbenchPrepare(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "sysbench", test.Prepare()...) + if test.FromScript { + lp := filepath.Join(scriptDir, luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) + } + return cmd +} + +// sysbenchRun returns a exec.Cmd for running the sysbench run step +func sysbenchRun(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "sysbench", test.Run()...) + if test.FromScript { + lp := filepath.Join(scriptDir, luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) + } + return cmd +} + +// sysbenchPrepare returns a exec.Cmd for running the sysbench cleanup step +func sysbenchCleanup(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { + cmd := ExecCommand(ctx, "sysbench", test.Cleanup()...) + if test.FromScript { + lp := filepath.Join(scriptDir, luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) + } + return cmd +} + +// benchmark runs a sysbench benchmark against a server calling prepare, run, cleanup +func benchmark( + ctx context.Context, + test *Test, + config *Config, + serverConfig *ServerConfig, + stampFunc func() string, + suiteId string, +) (*Result, error) { + prepare := sysbenchPrepare(ctx, test, config.ScriptDir) + run := sysbenchRun(ctx, test, config.ScriptDir) + cleanup := sysbenchCleanup(ctx, test, config.ScriptDir) + + fmt.Println("Running test ", test.Name) + + out, err := prepare.Output() + if err != nil { + fmt.Println(string(out)) + return nil, err + } + + out, err = run.Output() + if err != nil { + fmt.Print(string(out)) + return nil, err + } + + if Debug == true { + fmt.Print(string(out)) + } + + r, err := FromOutputResult(out, config, serverConfig, test, suiteId, nil) + if err != nil { + return nil, err + } + + r.Stamp(stampFunc) + + return r, cleanup.Run() +} + +//// fromChannelResults collects all Results from the given channel and returns them +//func fromChannelResults(rc chan *Result) Results { +// results := make(Results, 0) +// for r := range rc { +// if r != nil { +// results = append(results, r) +// } +// } +// return results +//} diff --git a/go/performance/utils/sysbench_runner/dolt_server.go b/go/performance/utils/sysbench_runner/dolt_server.go new file mode 100644 index 00000000000..bb6dd464794 --- /dev/null +++ b/go/performance/utils/sysbench_runner/dolt_server.go @@ -0,0 +1,97 @@ +package sysbench_runner + +import ( + "context" + "fmt" + "golang.org/x/sync/errgroup" + "os" + "os/exec" + "os/signal" + "sync" + "time" +) + +type doltServerImpl struct { + dir string + serverConfig *ServerConfig + serverCtx context.Context + serverCtxCancelFunc context.CancelFunc + server *exec.Cmd + serverEg *errgroup.Group + quit chan os.Signal + wg *sync.WaitGroup + killSignal os.Signal +} + +var _ Server = &doltServerImpl{} + +func NewDoltServer(ctx context.Context, dir string, serverConfig *ServerConfig, killSignal os.Signal, serverParams []string) *doltServerImpl { + withKeyCtx, cancel := context.WithCancel(ctx) + gServer, serverCtx := errgroup.WithContext(withKeyCtx) + + server := ExecCommand(serverCtx, serverConfig.ServerExec, serverParams...) + server.Dir = dir + + quit := make(chan os.Signal, 1) + return &doltServerImpl{ + dir: dir, + serverConfig: serverConfig, + server: server, + serverCtxCancelFunc: cancel, + serverEg: gServer, + quit: quit, + wg: &sync.WaitGroup{}, + killSignal: killSignal, + } +} + +func (s *doltServerImpl) Start(ctx context.Context) error { + signal.Notify(s.quit, os.Interrupt, s.killSignal) + s.wg.Add(1) + go func() { + <-s.quit + defer s.wg.Done() + signal.Stop(s.quit) + // todo: remove this ?? s.serverCtxCancelFunc() + }() + + // launch the dolt server + s.serverEg.Go(func() error { + return s.server.Run() + }) + // sleep to allow the server to start + time.Sleep(5 * time.Second) + return nil +} + +func (s *doltServerImpl) Stop(ctx context.Context) error { + defer s.serverCtxCancelFunc() + if s.wg != nil && s.serverEg != nil && s.serverCtx != nil && s.quit != nil { + // send signal to dolt server + s.quit <- s.killSignal + err := s.serverEg.Wait() + if err != nil { + // we expect a kill error + // we only exit in error if this is not the + // error + if err.Error() != "signal: killed" { + fmt.Println(err) + close(s.quit) + s.wg.Wait() + return err + } + } + + fmt.Println("Successfully killed server") + close(s.quit) + s.wg.Wait() + + s.wg = nil + s.quit = nil + s.serverCtx = nil + s.serverEg = nil + s.serverCtxCancelFunc = func() {} + } + + return nil +} diff --git a/go/performance/utils/sysbench_runner/run.go b/go/performance/utils/sysbench_runner/run.go index 5e1e764ccad..e1d4bb31fb8 100644 --- a/go/performance/utils/sysbench_runner/run.go +++ b/go/performance/utils/sysbench_runner/run.go @@ -35,6 +35,11 @@ func Run(config *Config) error { return err } + cwd, err := os.Getwd() + if err != nil { + return err + } + for _, serverConfig := range config.Servers { var results Results switch serverConfig.Server { @@ -45,7 +50,9 @@ func Run(config *Config) error { return ProfileDolt(ctx, config, serverConfig) } fmt.Println("Running dolt sysbench test") - results, err = BenchmarkDolt(ctx, config, serverConfig) + //results, err = BenchmarkDolt(ctx, config, serverConfig) + b := NewDoltBenchmarker(cwd, config, serverConfig) + results, err = b.Benchmark(ctx) case Doltgres: fmt.Println("Running doltgres sysbench test") results, err = BenchmarkDoltgres(ctx, config, serverConfig) diff --git a/go/performance/utils/sysbench_runner/server.go b/go/performance/utils/sysbench_runner/server.go new file mode 100644 index 00000000000..9c3d5de2f21 --- /dev/null +++ b/go/performance/utils/sysbench_runner/server.go @@ -0,0 +1,8 @@ +package sysbench_runner + +import "context" + +type Server interface { + Start(ctx context.Context) error + Stop(ctx context.Context) error +} diff --git a/go/performance/utils/sysbench_runner/sysbench.go b/go/performance/utils/sysbench_runner/sysbench.go new file mode 100644 index 00000000000..cc225b077e4 --- /dev/null +++ b/go/performance/utils/sysbench_runner/sysbench.go @@ -0,0 +1,192 @@ +package sysbench_runner + +import ( + "context" + "fmt" + "github.com/google/uuid" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + sysbenchCommand = "sysbench" + luaPathEnvVarTemplate = "LUA_PATH=%s" +) + +type SysbenchTester interface { + Test(ctx context.Context) (*Result, error) +} + +type sysbenchTestImpl struct { + test *Test + config *Config + scriptDir string + serverConfig *ServerConfig + stampFunc func() string + idFunc func() string + suiteId string +} + +var _ SysbenchTester = &sysbenchTestImpl{} + +func NewSysbenchTester(config *Config, serverConfig *ServerConfig, test *Test, stampFunc func() string) *sysbenchTestImpl { + return &sysbenchTestImpl{ + config: config, + serverConfig: serverConfig, + test: test, + suiteId: serverConfig.GetId(), + stampFunc: stampFunc, + } +} + +func (t *sysbenchTestImpl) newResult() (*Result, error) { + serverParams, err := t.serverConfig.GetServerArgs() + if err != nil { + return nil, err + } + + var getId func() string + if t.idFunc == nil { + getId = func() string { + return uuid.New().String() + } + } else { + getId = t.idFunc + } + + var name string + if t.test.FromScript { + base := filepath.Base(t.test.Name) + ext := filepath.Ext(base) + name = strings.TrimSuffix(base, ext) + } else { + name = t.test.Name + } + + return &Result{ + Id: getId(), + SuiteId: t.suiteId, + TestId: t.test.id, + RuntimeOS: t.config.RuntimeOS, + RuntimeGoArch: t.config.RuntimeGoArch, + ServerName: string(t.serverConfig.Server), + ServerVersion: t.serverConfig.Version, + ServerParams: strings.Join(serverParams, " "), + TestName: name, + TestParams: strings.Join(t.test.Params, " "), + }, nil +} + +func (t *sysbenchTestImpl) updateResult(result *Result, line string) error { + lineParts := strings.Split(line, ":") + key := strings.TrimSpace(lineParts[0]) + + if len(lineParts) > 1 { + rawVal := strings.TrimSpace(lineParts[1]) + err := updateResult(result, key, rawVal) + if err != nil { + return err + } + } + + return nil +} + +func (t *sysbenchTestImpl) outputToResult(output []byte) (*Result, error) { + result, err := t.newResult() + if err != nil { + return nil, err + } + + lines := strings.Split(string(output), "\n") + var process bool + for _, l := range lines { + trimmed := strings.TrimSpace(l) + if trimmed == "" { + continue + } + if strings.HasPrefix(trimmed, SqlStatsPrefix) { + process = true + continue + } + if process { + err := t.updateResult(result, trimmed) + if err != nil { + return result, err + } + } + } + + return result, nil +} + +func (t *sysbenchTestImpl) prepare(ctx context.Context) error { + cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.Prepare()...) + if t.test.FromScript { + lp := filepath.Join(t.scriptDir, luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) + } + out, err := cmd.Output() + if err != nil { + fmt.Println(string(out)) + return err + } + return nil +} + +func (t *sysbenchTestImpl) run(ctx context.Context) (*Result, error) { + cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.Run()...) + if t.test.FromScript { + lp := filepath.Join(t.scriptDir, luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) + } + + out, err := cmd.Output() + if err != nil { + fmt.Print(string(out)) + return nil, err + } + + if Debug == true { + fmt.Print(string(out)) + } + + rs, err := t.outputToResult(out) + if err != nil { + return nil, err + } + + rs.Stamp(t.stampFunc) + + return rs, nil +} + +func (t *sysbenchTestImpl) cleanup(ctx context.Context) error { + cmd := ExecCommand(ctx, sysbenchCommand, t.test.Cleanup()...) + if t.test.FromScript { + lp := filepath.Join(t.scriptDir, luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) + } + return cmd.Run() +} + +func (t *sysbenchTestImpl) Test(ctx context.Context) (*Result, error) { + err := t.prepare(ctx) + if err != nil { + return nil, err + } + + fmt.Println("Running test ", t.test.Name) + + rs, err := t.run(ctx) + if err != nil { + return nil, err + } + + return rs, t.cleanup(ctx) +} From 7931e910e30f8a4f06c1b081173d0c86674f8544 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Fri, 9 Feb 2024 15:37:50 -0800 Subject: [PATCH 03/25] /go/performance/utils/sysbench_runner: wip, still refactoring --- go/performance/utils/sysbench_runner/dolt.go | 2 +- .../utils/sysbench_runner/dolt_server.go | 97 -------- .../utils/sysbench_runner/doltgres.go | 215 ++++++++---------- .../utils/sysbench_runner/doltgres_old.go | 183 +++++++++++++++ go/performance/utils/sysbench_runner/run.go | 4 +- .../utils/sysbench_runner/server.go | 96 +++++++- 6 files changed, 372 insertions(+), 225 deletions(-) delete mode 100644 go/performance/utils/sysbench_runner/dolt_server.go create mode 100644 go/performance/utils/sysbench_runner/doltgres_old.go diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go index 1ddcce517f4..5bebbec23e5 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -125,7 +125,7 @@ func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { return nil, err } - server := NewDoltServer(ctx, testRepo, b.serverConfig, syscall.SIGTERM, serverParams) + server := NewServer(ctx, testRepo, b.serverConfig, syscall.SIGTERM, serverParams) err = server.Start(ctx) if err != nil { return nil, err diff --git a/go/performance/utils/sysbench_runner/dolt_server.go b/go/performance/utils/sysbench_runner/dolt_server.go deleted file mode 100644 index bb6dd464794..00000000000 --- a/go/performance/utils/sysbench_runner/dolt_server.go +++ /dev/null @@ -1,97 +0,0 @@ -package sysbench_runner - -import ( - "context" - "fmt" - "golang.org/x/sync/errgroup" - "os" - "os/exec" - "os/signal" - "sync" - "time" -) - -type doltServerImpl struct { - dir string - serverConfig *ServerConfig - serverCtx context.Context - serverCtxCancelFunc context.CancelFunc - server *exec.Cmd - serverEg *errgroup.Group - quit chan os.Signal - wg *sync.WaitGroup - killSignal os.Signal -} - -var _ Server = &doltServerImpl{} - -func NewDoltServer(ctx context.Context, dir string, serverConfig *ServerConfig, killSignal os.Signal, serverParams []string) *doltServerImpl { - withKeyCtx, cancel := context.WithCancel(ctx) - gServer, serverCtx := errgroup.WithContext(withKeyCtx) - - server := ExecCommand(serverCtx, serverConfig.ServerExec, serverParams...) - server.Dir = dir - - quit := make(chan os.Signal, 1) - return &doltServerImpl{ - dir: dir, - serverConfig: serverConfig, - server: server, - serverCtxCancelFunc: cancel, - serverEg: gServer, - quit: quit, - wg: &sync.WaitGroup{}, - killSignal: killSignal, - } -} - -func (s *doltServerImpl) Start(ctx context.Context) error { - signal.Notify(s.quit, os.Interrupt, s.killSignal) - s.wg.Add(1) - go func() { - <-s.quit - defer s.wg.Done() - signal.Stop(s.quit) - // todo: remove this ?? s.serverCtxCancelFunc() - }() - - // launch the dolt server - s.serverEg.Go(func() error { - return s.server.Run() - }) - // sleep to allow the server to start - time.Sleep(5 * time.Second) - return nil -} - -func (s *doltServerImpl) Stop(ctx context.Context) error { - defer s.serverCtxCancelFunc() - if s.wg != nil && s.serverEg != nil && s.serverCtx != nil && s.quit != nil { - // send signal to dolt server - s.quit <- s.killSignal - err := s.serverEg.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - fmt.Println(err) - close(s.quit) - s.wg.Wait() - return err - } - } - - fmt.Println("Successfully killed server") - close(s.quit) - s.wg.Wait() - - s.wg = nil - s.quit = nil - s.serverCtx = nil - s.serverEg = nil - s.serverCtxCancelFunc = func() {} - } - - return nil -} diff --git a/go/performance/utils/sysbench_runner/doltgres.go b/go/performance/utils/sysbench_runner/doltgres.go index 7364fa4efef..36c21e97c04 100644 --- a/go/performance/utils/sysbench_runner/doltgres.go +++ b/go/performance/utils/sysbench_runner/doltgres.go @@ -1,138 +1,78 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package sysbench_runner import ( "context" + "database/sql" "fmt" "os" - "os/signal" "path/filepath" - "sync" "syscall" - "time" - - "database/sql" +) - _ "github.com/lib/pq" - "golang.org/x/sync/errgroup" +const ( + postgresDriver = "postgres" + doltgresUser = "doltgres" + doltDataDir = ".dolt" + createDatabaseTemplate = "create database %s;" + psqlDsnTemplate = "host=%s port=%d user=%s password=%s dbname=%s sslmode=disable" ) -// BenchmarkDoltgres benchmarks doltgres based on the provided configurations -func BenchmarkDoltgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return nil, err - } +type doltgresBenchmarkerImpl struct { + dir string // cwd + config *Config + serverConfig *ServerConfig +} - err = DoltVersion(ctx, serverConfig.ServerExec) - if err != nil { - return nil, err - } +var _ Benchmarker = &doltgresBenchmarkerImpl{} - serverDir, err := createServerDir(dbName) - if err != nil { - return nil, err +func NewDoltgresBenchmarker(dir string, config *Config, serverConfig *ServerConfig) *doltgresBenchmarkerImpl { + return &doltgresBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, } - defer func() { - cleanupDoltgresServerDir(serverDir) - }() - - serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir)) - - withKeyCtx, cancel := context.WithCancel(ctx) - gServer, serverCtx := errgroup.WithContext(withKeyCtx) - - server := getServer(serverCtx, serverConfig, serverDir, serverParams) - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - // launch the dolt server - gServer.Go(func() error { - return server.Run() - }) +} - // sleep to allow the server to start - time.Sleep(5 * time.Second) +func (b *doltgresBenchmarkerImpl) checkInstallation(ctx context.Context) error { + version := ExecCommand(ctx, b.serverConfig.ServerExec, doltVersionCommand) + return version.Run() +} - // create the db against the running server - err = createDb(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "doltgres", dbName) +func (b *doltgresBenchmarkerImpl) createServerDir() (string, error) { + cwd, err := os.Getwd() if err != nil { - close(quit) - wg.Wait() - return nil, err + return "", err } - tests, err := GetTests(config, serverConfig, nil) + serverDir := filepath.Join(cwd, dbName) + err = os.MkdirAll(serverDir, os.ModePerm) if err != nil { - close(quit) - wg.Wait() - return nil, err + return "", err } - results := make(Results, 0) - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) + return serverDir, nil +} + +func (b *doltgresBenchmarkerImpl) cleanupServerDir(dir string) error { + dataDir := filepath.Join(dir, doltDataDir) + defaultDir := filepath.Join(dir, doltgresUser) + testDir := filepath.Join(dir, dbName) + for _, d := range []string{dataDir, defaultDir, testDir} { + if _, err := os.Stat(d); !os.IsNotExist(err) { + err = os.RemoveAll(d) if err != nil { - close(quit) - wg.Wait() - return nil, err + return err } - results = append(results, r) - } - } - - // send signal to dolt server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - fmt.Println(err) - close(quit) - wg.Wait() - return nil, err } } - - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - return results, nil + return nil } -func createDb(ctx context.Context, host, port, user, dbname string) error { - psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname) +func (b *doltgresBenchmarkerImpl) createTestingDb(ctx context.Context) error { + psqlconn := fmt.Sprintf(psqlDsnTemplate, b.serverConfig.Host, b.serverConfig.Port, doltgresUser, "", dbName) // open database - db, err := sql.Open("postgres", psqlconn) + db, err := sql.Open(postgresDriver, psqlconn) if err != nil { return err } @@ -146,38 +86,63 @@ func createDb(ctx context.Context, host, port, user, dbname string) error { return err } - _, err = db.ExecContext(ctx, fmt.Sprintf("create database %s;", dbname)) + _, err = db.ExecContext(ctx, fmt.Sprintf(createDatabaseTemplate, dbName)) return err } -// createServerDir creates a server directory -func createServerDir(dbName string) (string, error) { - cwd, err := os.Getwd() +func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { + err := b.checkInstallation(ctx) if err != nil { - return "", err + return nil, err } - serverDir := filepath.Join(cwd, dbName) - err = os.MkdirAll(serverDir, os.ModePerm) + serverDir, err := createServerDir(dbName) if err != nil { - return "", err + return nil, err } + defer func() { + cleanupDoltgresServerDir(serverDir) + }() - return serverDir, nil -} + serverParams, err := b.serverConfig.GetServerArgs() + if err != nil { + return nil, err + } + serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir)) -// cleanupDoltgresServerDir cleans up the doltgres assets in the provided dir -func cleanupDoltgresServerDir(dir string) error { - dataDir := filepath.Join(dir, ".dolt") - defaultDir := filepath.Join(dir, "doltgres") - testDir := filepath.Join(dir, dbName) - for _, d := range []string{dataDir, defaultDir, testDir} { - if _, err := os.Stat(d); !os.IsNotExist(err) { - err = os.RemoveAll(d) + server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start(ctx) + if err != nil { + return nil, err + } + + err = b.createTestingDb(ctx) + if err != nil { + return nil, err + } + + tests, err := GetTests(b.config, b.serverConfig, nil) + if err != nil { + return nil, err + } + + results := make(Results, 0) + for i := 0; i < b.config.Runs; i++ { + for _, test := range tests { + tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) + r, err := tester.Test(ctx) if err != nil { - return err + server.Stop(ctx) + return nil, err } + results = append(results, r) } } - return nil + + err = server.Stop(ctx) + if err != nil { + return nil, err + } + + return results, nil } diff --git a/go/performance/utils/sysbench_runner/doltgres_old.go b/go/performance/utils/sysbench_runner/doltgres_old.go new file mode 100644 index 00000000000..7364fa4efef --- /dev/null +++ b/go/performance/utils/sysbench_runner/doltgres_old.go @@ -0,0 +1,183 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "context" + "fmt" + "os" + "os/signal" + "path/filepath" + "sync" + "syscall" + "time" + + "database/sql" + + _ "github.com/lib/pq" + "golang.org/x/sync/errgroup" +) + +// BenchmarkDoltgres benchmarks doltgres based on the provided configurations +func BenchmarkDoltgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { + serverParams, err := serverConfig.GetServerArgs() + if err != nil { + return nil, err + } + + err = DoltVersion(ctx, serverConfig.ServerExec) + if err != nil { + return nil, err + } + + serverDir, err := createServerDir(dbName) + if err != nil { + return nil, err + } + defer func() { + cleanupDoltgresServerDir(serverDir) + }() + + serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir)) + + withKeyCtx, cancel := context.WithCancel(ctx) + gServer, serverCtx := errgroup.WithContext(withKeyCtx) + + server := getServer(serverCtx, serverConfig, serverDir, serverParams) + + // handle user interrupt + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt, syscall.SIGTERM) + var wg sync.WaitGroup + wg.Add(1) + go func() { + <-quit + defer wg.Done() + signal.Stop(quit) + cancel() + }() + + // launch the dolt server + gServer.Go(func() error { + return server.Run() + }) + + // sleep to allow the server to start + time.Sleep(5 * time.Second) + + // create the db against the running server + err = createDb(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "doltgres", dbName) + if err != nil { + close(quit) + wg.Wait() + return nil, err + } + + tests, err := GetTests(config, serverConfig, nil) + if err != nil { + close(quit) + wg.Wait() + return nil, err + } + + results := make(Results, 0) + for i := 0; i < config.Runs; i++ { + for _, test := range tests { + r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) + if err != nil { + close(quit) + wg.Wait() + return nil, err + } + results = append(results, r) + } + } + + // send signal to dolt server + quit <- syscall.SIGTERM + + err = gServer.Wait() + if err != nil { + // we expect a kill error + // we only exit in error if this is not the + // error + if err.Error() != "signal: killed" { + fmt.Println(err) + close(quit) + wg.Wait() + return nil, err + } + } + + fmt.Println("Successfully killed server") + close(quit) + wg.Wait() + + return results, nil +} + +func createDb(ctx context.Context, host, port, user, dbname string) error { + psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname) + + // open database + db, err := sql.Open("postgres", psqlconn) + if err != nil { + return err + } + + // close database + defer db.Close() + + // check db + err = db.PingContext(ctx) + if err != nil { + return err + } + + _, err = db.ExecContext(ctx, fmt.Sprintf("create database %s;", dbname)) + return err +} + +// createServerDir creates a server directory +func createServerDir(dbName string) (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + + serverDir := filepath.Join(cwd, dbName) + err = os.MkdirAll(serverDir, os.ModePerm) + if err != nil { + return "", err + } + + return serverDir, nil +} + +// cleanupDoltgresServerDir cleans up the doltgres assets in the provided dir +func cleanupDoltgresServerDir(dir string) error { + dataDir := filepath.Join(dir, ".dolt") + defaultDir := filepath.Join(dir, "doltgres") + testDir := filepath.Join(dir, dbName) + for _, d := range []string{dataDir, defaultDir, testDir} { + if _, err := os.Stat(d); !os.IsNotExist(err) { + err = os.RemoveAll(d) + if err != nil { + return err + } + } + } + return nil +} diff --git a/go/performance/utils/sysbench_runner/run.go b/go/performance/utils/sysbench_runner/run.go index e1d4bb31fb8..9744e2f09e9 100644 --- a/go/performance/utils/sysbench_runner/run.go +++ b/go/performance/utils/sysbench_runner/run.go @@ -55,7 +55,9 @@ func Run(config *Config) error { results, err = b.Benchmark(ctx) case Doltgres: fmt.Println("Running doltgres sysbench test") - results, err = BenchmarkDoltgres(ctx, config, serverConfig) + b := NewDoltgresBenchmarker(cwd, config, serverConfig) + results, err = b.Benchmark(ctx) + //results, err = BenchmarkDoltgres(ctx, config, serverConfig) case MySql: fmt.Println("Running mysql sysbench test") results, err = BenchmarkMysql(ctx, config, serverConfig) diff --git a/go/performance/utils/sysbench_runner/server.go b/go/performance/utils/sysbench_runner/server.go index 9c3d5de2f21..dac67a972db 100644 --- a/go/performance/utils/sysbench_runner/server.go +++ b/go/performance/utils/sysbench_runner/server.go @@ -1,8 +1,102 @@ package sysbench_runner -import "context" +import ( + "context" + "fmt" + "golang.org/x/sync/errgroup" + "os" + "os/exec" + "os/signal" + "sync" + "time" +) type Server interface { Start(ctx context.Context) error Stop(ctx context.Context) error } + +type doltServerImpl struct { + dir string + serverConfig *ServerConfig + serverCtx context.Context + serverCtxCancelFunc context.CancelFunc + server *exec.Cmd + serverEg *errgroup.Group + quit chan os.Signal + wg *sync.WaitGroup + killSignal os.Signal +} + +var _ Server = &doltServerImpl{} + +func NewServer(ctx context.Context, dir string, serverConfig *ServerConfig, killSignal os.Signal, serverParams []string) *doltServerImpl { + withKeyCtx, cancel := context.WithCancel(ctx) + gServer, serverCtx := errgroup.WithContext(withKeyCtx) + + server := ExecCommand(serverCtx, serverConfig.ServerExec, serverParams...) + server.Dir = dir + + quit := make(chan os.Signal, 1) + return &doltServerImpl{ + dir: dir, + serverConfig: serverConfig, + server: server, + serverCtxCancelFunc: cancel, + serverEg: gServer, + quit: quit, + wg: &sync.WaitGroup{}, + killSignal: killSignal, + } +} + +func (s *doltServerImpl) Start(ctx context.Context) error { + signal.Notify(s.quit, os.Interrupt, s.killSignal) + s.wg.Add(1) + go func() { + <-s.quit + defer s.wg.Done() + signal.Stop(s.quit) + // todo: remove this ?? s.serverCtxCancelFunc() + }() + + // launch the dolt server + s.serverEg.Go(func() error { + return s.server.Run() + }) + // sleep to allow the server to start + time.Sleep(10 * time.Second) + return nil +} + +func (s *doltServerImpl) Stop(ctx context.Context) error { + defer s.serverCtxCancelFunc() + if s.wg != nil && s.serverEg != nil && s.serverCtx != nil && s.quit != nil { + // send signal to dolt server + s.quit <- s.killSignal + err := s.serverEg.Wait() + if err != nil { + // we expect a kill error + // we only exit in error if this is not the + // error + if err.Error() != "signal: killed" { + fmt.Println(err) + close(s.quit) + s.wg.Wait() + return err + } + } + + fmt.Println("Successfully killed server") + close(s.quit) + s.wg.Wait() + + s.wg = nil + s.quit = nil + s.serverCtx = nil + s.serverEg = nil + s.serverCtxCancelFunc = func() {} + } + + return nil +} From 4894a27d6b2be734f7272e452c51cf10fc42c536 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Fri, 9 Feb 2024 16:06:58 -0800 Subject: [PATCH 04/25] /go/performance/utils/sysbench_runner: wip, still need to finish postgres and stuff --- .../utils/sysbench_runner/doltgres.go | 15 +- .../utils/sysbench_runner/doltgres_old.go | 6 +- go/performance/utils/sysbench_runner/mysql.go | 290 +++++++----------- .../utils/sysbench_runner/mysql_old.go | 248 +++++++++++++++ .../utils/sysbench_runner/postgres.go | 2 +- go/performance/utils/sysbench_runner/run.go | 4 +- 6 files changed, 363 insertions(+), 202 deletions(-) create mode 100644 go/performance/utils/sysbench_runner/mysql_old.go diff --git a/go/performance/utils/sysbench_runner/doltgres.go b/go/performance/utils/sysbench_runner/doltgres.go index 36c21e97c04..94e64c9a04e 100644 --- a/go/performance/utils/sysbench_runner/doltgres.go +++ b/go/performance/utils/sysbench_runner/doltgres.go @@ -39,18 +39,7 @@ func (b *doltgresBenchmarkerImpl) checkInstallation(ctx context.Context) error { } func (b *doltgresBenchmarkerImpl) createServerDir() (string, error) { - cwd, err := os.Getwd() - if err != nil { - return "", err - } - - serverDir := filepath.Join(cwd, dbName) - err = os.MkdirAll(serverDir, os.ModePerm) - if err != nil { - return "", err - } - - return serverDir, nil + return CreateServerDir(dbName) } func (b *doltgresBenchmarkerImpl) cleanupServerDir(dir string) error { @@ -96,7 +85,7 @@ func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error return nil, err } - serverDir, err := createServerDir(dbName) + serverDir, err := CreateServerDir(dbName) if err != nil { return nil, err } diff --git a/go/performance/utils/sysbench_runner/doltgres_old.go b/go/performance/utils/sysbench_runner/doltgres_old.go index 7364fa4efef..24bf05e3c4b 100644 --- a/go/performance/utils/sysbench_runner/doltgres_old.go +++ b/go/performance/utils/sysbench_runner/doltgres_old.go @@ -42,7 +42,7 @@ func BenchmarkDoltgres(ctx context.Context, config *Config, serverConfig *Server return nil, err } - serverDir, err := createServerDir(dbName) + serverDir, err := CreateServerDir(dbName) if err != nil { return nil, err } @@ -150,8 +150,8 @@ func createDb(ctx context.Context, host, port, user, dbname string) error { return err } -// createServerDir creates a server directory -func createServerDir(dbName string) (string, error) { +// CreateServerDir creates a server directory +func CreateServerDir(dbName string) (string, error) { cwd, err := os.Getwd() if err != nil { return "", err diff --git a/go/performance/utils/sysbench_runner/mysql.go b/go/performance/utils/sysbench_runner/mysql.go index 7a802a92099..82735bf4056 100644 --- a/go/performance/utils/sysbench_runner/mysql.go +++ b/go/performance/utils/sysbench_runner/mysql.go @@ -1,164 +1,55 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package sysbench_runner import ( "context" "database/sql" "fmt" + _ "github.com/go-sql-driver/mysql" "log" "os" - "os/exec" - "os/signal" - "sync" "syscall" - "time" +) - _ "github.com/go-sql-driver/mysql" - "golang.org/x/sync/errgroup" +const ( + mysqlDriverName = "mysql" + mysqlRootTCPDsnTemplate = "root@tcp(%s:%d)/" + mysqlRootUnixDsnTemplate = "root@unix(%s)/" + mysqlDropDatabaseSqlTemplate = "DROP DATABASE IF EXISTS %s;" + mysqlCreateDatabaseSqlTemplate = "CREATE DATABASE %s;" + mysqlDropUserSqlTemplate = "DROP USER IF EXISTS %s;" + mysqlCreateUserSqlTemplate = "CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s';" + mysqlGrantPermissionsSqlTemplate = "GRANT ALL ON %s.* to %s;" + mysqlSetGlobalLocalInfileSql = "SET GLOBAL local_infile = 'ON';" + mysqlSetGlobalSqlModeSql = "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" ) -type MysqlConfig struct { - Socket string - ConnectionProtocol string - Port int - Host string +type mysqlBenchmarkerImpl struct { + dir string // cwd + config *Config + serverConfig *ServerConfig } -// BenchmarkMysql benchmarks mysql based on the provided configurations -func BenchmarkMysql(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - withKeyCtx, cancel := context.WithCancel(ctx) - - var serverDir string - defer func() { - if serverDir != "" { - os.RemoveAll(serverDir) - } - }() - - var localServer bool - var gServer *errgroup.Group - var serverCtx context.Context - var server *exec.Cmd - var err error - if serverConfig.Host == defaultHost { - log.Println("Launching the default server") - localServer = true - - serverDir, err = InitMysqlDataDir(ctx, serverConfig) - if err != nil { - cancel() - return nil, err - } - - gServer, serverCtx = errgroup.WithContext(withKeyCtx) - var serverParams []string - serverParams, err = serverConfig.GetServerArgs() - if err != nil { - cancel() - return nil, err - } - serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) - - server = getMysqlServer(serverCtx, serverConfig, serverParams) - - // launch the mysql server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(10 * time.Second) - - // setup mysqldb - err := SetupDB(ctx, GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName) - if err != nil { - cancel() - return nil, err - } - log.Println("Successfully set up the MySQL database") - } - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - tests, err := GetTests(config, serverConfig, nil) - if err != nil { - return nil, err - } +var _ Benchmarker = &mysqlBenchmarkerImpl{} - results := make(Results, 0) - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - results = append(results, r) - } +func NewMysqlBenchmarker(dir string, config *Config, serverConfig *ServerConfig) *mysqlBenchmarkerImpl { + return &mysqlBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, } - - // stop local mysql server - if localServer { - // send signal to server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - close(quit) - wg.Wait() - return nil, err - } - } - } - - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - return results, nil } -// getMysqlServer returns a exec.Cmd for a dolt server -func getMysqlServer(ctx context.Context, config *ServerConfig, params []string) *exec.Cmd { - return ExecCommand(ctx, config.ServerExec, params...) +func (b *mysqlBenchmarkerImpl) createServerDir() (string, error) { + return CreateServerDir(dbName) } -// InitMysqlDataDir initializes a mysql data dir and returns the path -func InitMysqlDataDir(ctx context.Context, config *ServerConfig) (string, error) { - serverDir, err := createServerDir(dbName) +func (b *mysqlBenchmarkerImpl) initMysqlDataDir(ctx context.Context) (string, error) { + serverDir, err := b.createServerDir() if err != nil { return "", err } - msInit := ExecCommand(ctx, config.ServerExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) + msInit := ExecCommand(ctx, b.serverConfig.ServerExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) err = msInit.Run() if err != nil { return "", err @@ -167,16 +58,34 @@ func InitMysqlDataDir(ctx context.Context, config *ServerConfig) (string, error) return serverDir, nil } -func SetupDB(ctx context.Context, mConfig MysqlConfig, databaseName string) (err error) { - dsn, err := FormatDsn(mConfig) +func (b *mysqlBenchmarkerImpl) getDsn() (string, error) { + var socketPath string + if b.serverConfig.Socket != "" { + socketPath = b.serverConfig.Socket + } else { + socketPath = defaultMysqlSocket + } + + if b.serverConfig.ConnectionProtocol == tcpProtocol { + return fmt.Sprintf(mysqlRootTCPDsnTemplate, b.serverConfig.Host, b.serverConfig.Port), nil + } else if b.serverConfig.ConnectionProtocol == unixProtocol { + return fmt.Sprintf(mysqlRootUnixDsnTemplate, socketPath), nil + } else { + return "", ErrUnsupportedConnectionProtocol + } +} + +func (b *mysqlBenchmarkerImpl) createTestingDb(ctx context.Context) (err error) { + var dsn string + dsn, err = b.getDsn() if err != nil { - return err + return } - // TODO make sure this can work on windows - db, err := sql.Open("mysql", dsn) + var db *sql.DB + db, err = sql.Open(mysqlDriverName, dsn) if err != nil { - return err + return } defer func() { rerr := db.Close() @@ -184,65 +93,78 @@ func SetupDB(ctx context.Context, mConfig MysqlConfig, databaseName string) (err err = rerr } }() + err = db.Ping() if err != nil { - return err + return } - _, err = db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", databaseName)) - if err != nil { - return err + + stmts := []string{ + fmt.Sprintf(mysqlDropDatabaseSqlTemplate, dbName), + fmt.Sprintf(mysqlCreateDatabaseSqlTemplate, dbName), + fmt.Sprintf(mysqlDropUserSqlTemplate, sysbenchUserLocal), + fmt.Sprintf(mysqlCreateUserSqlTemplate, sysbenchUserLocal, sysbenchPassLocal), + fmt.Sprintf(mysqlGrantPermissionsSqlTemplate, dbName, sysbenchUserLocal), + mysqlSetGlobalLocalInfileSql, + mysqlSetGlobalSqlModeSql, // Required for running groupby_scan.lua without error } - _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE %s", databaseName)) - if err != nil { - return err + + for _, s := range stmts { + _, err = db.ExecContext(ctx, s) + if err != nil { + return + } } - _, err = db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s", sysbenchUserLocal)) + + log.Println("Successfully set up the MySQL database") + return +} + +func (b *mysqlBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { + serverDir, err := b.initMysqlDataDir(ctx) if err != nil { - return err + return nil, err } - _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s'", sysbenchUserLocal, sysbenchPassLocal)) + + serverParams, err := b.serverConfig.GetServerArgs() if err != nil { - return err + return nil, err } - _, err = db.ExecContext(ctx, fmt.Sprintf("GRANT ALL ON %s.* to %s", databaseName, sysbenchUserLocal)) + serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) + + server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start(ctx) if err != nil { - return err + return nil, err } - _, err = db.ExecContext(ctx, "SET GLOBAL local_infile = 'ON'") + + err = b.createTestingDb(ctx) if err != nil { - return err + return nil, err } - // Required for running groupby_scan.lua without error - _, err = db.ExecContext(ctx, "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));") + + tests, err := GetTests(b.config, b.serverConfig, nil) if err != nil { - return err + return nil, err } - return -} - -func FormatDsn(mConfig MysqlConfig) (string, error) { - var socketPath string - if mConfig.Socket != "" { - socketPath = mConfig.Socket - } else { - socketPath = defaultMysqlSocket + results := make(Results, 0) + for i := 0; i < b.config.Runs; i++ { + for _, test := range tests { + tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop(ctx) + return nil, err + } + results = append(results, r) + } } - if mConfig.ConnectionProtocol == tcpProtocol { - return fmt.Sprintf("root@tcp(%s:%d)/", mConfig.Host, mConfig.Port), nil - } else if mConfig.ConnectionProtocol == unixProtocol { - return fmt.Sprintf("root@unix(%s)/", socketPath), nil - } else { - return "", ErrUnsupportedConnectionProtocol + err = server.Stop(ctx) + if err != nil { + return nil, err } -} -func GetMysqlConnectionConfigFromServerConfig(config *ServerConfig) MysqlConfig { - return MysqlConfig{ - Socket: config.Socket, - ConnectionProtocol: config.ConnectionProtocol, - Port: config.Port, - Host: defaultHost, - } + return results, os.RemoveAll(serverDir) } diff --git a/go/performance/utils/sysbench_runner/mysql_old.go b/go/performance/utils/sysbench_runner/mysql_old.go new file mode 100644 index 00000000000..e6f66a791c6 --- /dev/null +++ b/go/performance/utils/sysbench_runner/mysql_old.go @@ -0,0 +1,248 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "context" + "database/sql" + "fmt" + "log" + "os" + "os/exec" + "os/signal" + "sync" + "syscall" + "time" + + _ "github.com/go-sql-driver/mysql" + "golang.org/x/sync/errgroup" +) + +type MysqlConfig struct { + Socket string + ConnectionProtocol string + Port int + Host string +} + +// BenchmarkMysql benchmarks mysql based on the provided configurations +func BenchmarkMysql(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { + withKeyCtx, cancel := context.WithCancel(ctx) + + var serverDir string + defer func() { + if serverDir != "" { + os.RemoveAll(serverDir) + } + }() + + var localServer bool + var gServer *errgroup.Group + var serverCtx context.Context + var server *exec.Cmd + var err error + if serverConfig.Host == defaultHost { + log.Println("Launching the default server") + localServer = true + + serverDir, err = InitMysqlDataDir(ctx, serverConfig) + if err != nil { + cancel() + return nil, err + } + + gServer, serverCtx = errgroup.WithContext(withKeyCtx) + var serverParams []string + serverParams, err = serverConfig.GetServerArgs() + if err != nil { + cancel() + return nil, err + } + serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) + + server = getMysqlServer(serverCtx, serverConfig, serverParams) + + // launch the mysql server + gServer.Go(func() error { + return server.Run() + }) + + // sleep to allow the server to start + time.Sleep(10 * time.Second) + + // setup mysqldb + err := SetupDB(ctx, GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName) + if err != nil { + cancel() + return nil, err + } + log.Println("Successfully set up the MySQL database") + } + + // handle user interrupt + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt, syscall.SIGTERM) + var wg sync.WaitGroup + wg.Add(1) + go func() { + <-quit + defer wg.Done() + signal.Stop(quit) + cancel() + }() + + tests, err := GetTests(config, serverConfig, nil) + if err != nil { + return nil, err + } + + results := make(Results, 0) + for i := 0; i < config.Runs; i++ { + for _, test := range tests { + r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) + if err != nil { + close(quit) + wg.Wait() + return nil, err + } + results = append(results, r) + } + } + + // stop local mysql server + if localServer { + // send signal to server + quit <- syscall.SIGTERM + + err = gServer.Wait() + if err != nil { + // we expect a kill error + // we only exit in error if this is not the + // error + if err.Error() != "signal: killed" { + close(quit) + wg.Wait() + return nil, err + } + } + } + + fmt.Println("Successfully killed server") + close(quit) + wg.Wait() + + return results, nil +} + +// getMysqlServer returns a exec.Cmd for a dolt server +func getMysqlServer(ctx context.Context, config *ServerConfig, params []string) *exec.Cmd { + return ExecCommand(ctx, config.ServerExec, params...) +} + +// InitMysqlDataDir initializes a mysql data dir and returns the path +func InitMysqlDataDir(ctx context.Context, config *ServerConfig) (string, error) { + serverDir, err := CreateServerDir(dbName) + if err != nil { + return "", err + } + + msInit := ExecCommand(ctx, config.ServerExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) + err = msInit.Run() + if err != nil { + return "", err + } + + return serverDir, nil +} + +func SetupDB(ctx context.Context, mConfig MysqlConfig, databaseName string) (err error) { + dsn, err := FormatDsn(mConfig) + if err != nil { + return err + } + + // TODO make sure this can work on windows + db, err := sql.Open("mysql", dsn) + if err != nil { + return err + } + defer func() { + rerr := db.Close() + if err == nil { + err = rerr + } + }() + err = db.Ping() + if err != nil { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", databaseName)) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE %s", databaseName)) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s", sysbenchUserLocal)) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s'", sysbenchUserLocal, sysbenchPassLocal)) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("GRANT ALL ON %s.* to %s", databaseName, sysbenchUserLocal)) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, "SET GLOBAL local_infile = 'ON'") + if err != nil { + return err + } + // Required for running groupby_scan.lua without error + _, err = db.ExecContext(ctx, "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));") + if err != nil { + return err + } + + return +} + +func FormatDsn(mConfig MysqlConfig) (string, error) { + var socketPath string + if mConfig.Socket != "" { + socketPath = mConfig.Socket + } else { + socketPath = defaultMysqlSocket + } + + if mConfig.ConnectionProtocol == tcpProtocol { + return fmt.Sprintf("root@tcp(%s:%d)/", mConfig.Host, mConfig.Port), nil + } else if mConfig.ConnectionProtocol == unixProtocol { + return fmt.Sprintf("root@unix(%s)/", socketPath), nil + } else { + return "", ErrUnsupportedConnectionProtocol + } +} + +func GetMysqlConnectionConfigFromServerConfig(config *ServerConfig) MysqlConfig { + return MysqlConfig{ + Socket: config.Socket, + ConnectionProtocol: config.ConnectionProtocol, + Port: config.Port, + Host: defaultHost, + } +} diff --git a/go/performance/utils/sysbench_runner/postgres.go b/go/performance/utils/sysbench_runner/postgres.go index ce2e26be82e..0e2a0323270 100644 --- a/go/performance/utils/sysbench_runner/postgres.go +++ b/go/performance/utils/sysbench_runner/postgres.go @@ -149,7 +149,7 @@ func BenchmarkPostgres(ctx context.Context, config *Config, serverConfig *Server // initPostgresDataDir initializes a postgres data dir and returns the path func initPostgresDataDir(ctx context.Context, config *ServerConfig) (string, error) { - serverDir, err := createServerDir(dbName) + serverDir, err := CreateServerDir(dbName) if err != nil { return "", err } diff --git a/go/performance/utils/sysbench_runner/run.go b/go/performance/utils/sysbench_runner/run.go index 9744e2f09e9..45010dbddb3 100644 --- a/go/performance/utils/sysbench_runner/run.go +++ b/go/performance/utils/sysbench_runner/run.go @@ -60,7 +60,9 @@ func Run(config *Config) error { //results, err = BenchmarkDoltgres(ctx, config, serverConfig) case MySql: fmt.Println("Running mysql sysbench test") - results, err = BenchmarkMysql(ctx, config, serverConfig) + b := NewMysqlBenchmarker(cwd, config, serverConfig) + results, err = b.Benchmark(ctx) + //results, err = BenchmarkMysql(ctx, config, serverConfig) case Postgres: fmt.Println("Running postgres sysbench test") results, err = BenchmarkPostgres(ctx, config, serverConfig) From 57afbe974bdb1aec3315d0b271e1c4cadf87f7ba Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 12 Feb 2024 11:41:40 -0800 Subject: [PATCH 05/25] /go/performance/utils/sysbench_runner: wip, dolt and mysql seem to work, testing postgres and doltgres --- go/performance/utils/sysbench_runner/dolt.go | 14 +- .../utils/sysbench_runner/dolt_old.go | 34 +- .../utils/sysbench_runner/doltgres.go | 18 +- .../utils/sysbench_runner/doltgres_old.go | 304 ++++++++---------- go/performance/utils/sysbench_runner/mysql.go | 5 +- .../utils/sysbench_runner/mysql_old.go | 232 +++++++------ .../utils/sysbench_runner/postgres.go | 237 +++++--------- .../utils/sysbench_runner/postgres_old.go | 187 +++++++++++ go/performance/utils/sysbench_runner/run.go | 25 +- .../utils/sysbench_runner/server.go | 34 +- .../utils/sysbench_runner/sysbench.go | 13 +- 11 files changed, 603 insertions(+), 500 deletions(-) create mode 100644 go/performance/utils/sysbench_runner/postgres_old.go diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go index 5bebbec23e5..620efb6fd6f 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -2,10 +2,12 @@ package sysbench_runner import ( "context" - "github.com/dolthub/dolt/go/store/types" "os" "path/filepath" "syscall" + "time" + + "github.com/dolthub/dolt/go/store/types" ) const ( @@ -25,6 +27,8 @@ const ( nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" ) +var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } + type doltBenchmarkerImpl struct { dir string // cwd config *Config @@ -42,7 +46,7 @@ func NewDoltBenchmarker(dir string, config *Config, serverConfig *ServerConfig) } // checkSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary -func (b *doltBenchmarkerImpl) checkSetDoltConfig(ctx context.Context, serverExec, key, val string) error { +func (b *doltBenchmarkerImpl) checkSetDoltConfig(ctx context.Context, key, val string) error { check := ExecCommand(ctx, b.serverConfig.ServerExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigGetFlag, key) err := check.Run() if err != nil { @@ -50,7 +54,7 @@ func (b *doltBenchmarkerImpl) checkSetDoltConfig(ctx context.Context, serverExec if err.Error() != "exit status 1" { return err } - set := ExecCommand(ctx, serverExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigAddFlag, key, val) + set := ExecCommand(ctx, b.serverConfig.ServerExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigAddFlag, key, val) err := set.Run() if err != nil { return err @@ -60,11 +64,11 @@ func (b *doltBenchmarkerImpl) checkSetDoltConfig(ctx context.Context, serverExec } func (b *doltBenchmarkerImpl) updateGlobalConfig(ctx context.Context) error { - err := checkSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigUsernameKey, doltBenchmarkUser) + err := b.checkSetDoltConfig(ctx, doltConfigUsernameKey, doltBenchmarkUser) if err != nil { return err } - return checkSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigEmailKey, doltBenchmarkEmail) + return b.checkSetDoltConfig(ctx, doltConfigEmailKey, doltBenchmarkEmail) } func (b *doltBenchmarkerImpl) checkInstallation(ctx context.Context) error { diff --git a/go/performance/utils/sysbench_runner/dolt_old.go b/go/performance/utils/sysbench_runner/dolt_old.go index 876bb2f11a9..deafc05daa7 100644 --- a/go/performance/utils/sysbench_runner/dolt_old.go +++ b/go/performance/utils/sysbench_runner/dolt_old.go @@ -17,22 +17,20 @@ package sysbench_runner import ( "context" "fmt" + "github.com/dolthub/dolt/go/store/types" "os" "os/exec" "path/filepath" - "time" - - "github.com/dolthub/dolt/go/store/types" ) const ( - //dbName = "test" - luaPath = "?.lua" - //bigEmptyRepo = "max-hoffman/big-empty" - //nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" +// dbName = "test" +// luaPath = "?.lua" +// bigEmptyRepo = "max-hoffman/big-empty" +// nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" ) -var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } +//var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } //// BenchmarkDolt benchmarks dolt based on the provided configurations //func BenchmarkDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { @@ -279,13 +277,13 @@ func benchmark( return r, cleanup.Run() } -//// fromChannelResults collects all Results from the given channel and returns them -//func fromChannelResults(rc chan *Result) Results { -// results := make(Results, 0) -// for r := range rc { -// if r != nil { -// results = append(results, r) -// } -// } -// return results -//} +// fromChannelResults collects all Results from the given channel and returns them +func fromChannelResults(rc chan *Result) Results { + results := make(Results, 0) + for r := range rc { + if r != nil { + results = append(results, r) + } + } + return results +} diff --git a/go/performance/utils/sysbench_runner/doltgres.go b/go/performance/utils/sysbench_runner/doltgres.go index 94e64c9a04e..7d1e1ecc8f5 100644 --- a/go/performance/utils/sysbench_runner/doltgres.go +++ b/go/performance/utils/sysbench_runner/doltgres.go @@ -90,7 +90,7 @@ func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error return nil, err } defer func() { - cleanupDoltgresServerDir(serverDir) + b.cleanupServerDir(serverDir) }() serverParams, err := b.serverConfig.GetServerArgs() @@ -135,3 +135,19 @@ func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error return results, nil } + +// CreateServerDir creates a server directory +func CreateServerDir(dbName string) (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + + serverDir := filepath.Join(cwd, dbName) + err = os.MkdirAll(serverDir, os.ModePerm) + if err != nil { + return "", err + } + + return serverDir, nil +} diff --git a/go/performance/utils/sysbench_runner/doltgres_old.go b/go/performance/utils/sysbench_runner/doltgres_old.go index 24bf05e3c4b..dfa8a1dc0ea 100644 --- a/go/performance/utils/sysbench_runner/doltgres_old.go +++ b/go/performance/utils/sysbench_runner/doltgres_old.go @@ -14,170 +14,142 @@ package sysbench_runner -import ( - "context" - "fmt" - "os" - "os/signal" - "path/filepath" - "sync" - "syscall" - "time" - - "database/sql" - - _ "github.com/lib/pq" - "golang.org/x/sync/errgroup" -) - -// BenchmarkDoltgres benchmarks doltgres based on the provided configurations -func BenchmarkDoltgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return nil, err - } - - err = DoltVersion(ctx, serverConfig.ServerExec) - if err != nil { - return nil, err - } - - serverDir, err := CreateServerDir(dbName) - if err != nil { - return nil, err - } - defer func() { - cleanupDoltgresServerDir(serverDir) - }() - - serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir)) - - withKeyCtx, cancel := context.WithCancel(ctx) - gServer, serverCtx := errgroup.WithContext(withKeyCtx) - - server := getServer(serverCtx, serverConfig, serverDir, serverParams) - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - // launch the dolt server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(5 * time.Second) - - // create the db against the running server - err = createDb(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "doltgres", dbName) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - - tests, err := GetTests(config, serverConfig, nil) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - - results := make(Results, 0) - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - results = append(results, r) - } - } - - // send signal to dolt server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - fmt.Println(err) - close(quit) - wg.Wait() - return nil, err - } - } - - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - return results, nil -} - -func createDb(ctx context.Context, host, port, user, dbname string) error { - psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname) - - // open database - db, err := sql.Open("postgres", psqlconn) - if err != nil { - return err - } - - // close database - defer db.Close() - - // check db - err = db.PingContext(ctx) - if err != nil { - return err - } - - _, err = db.ExecContext(ctx, fmt.Sprintf("create database %s;", dbname)) - return err -} - -// CreateServerDir creates a server directory -func CreateServerDir(dbName string) (string, error) { - cwd, err := os.Getwd() - if err != nil { - return "", err - } - - serverDir := filepath.Join(cwd, dbName) - err = os.MkdirAll(serverDir, os.ModePerm) - if err != nil { - return "", err - } - - return serverDir, nil -} +//import ( +// _ "github.com/lib/pq" +//) + +//// BenchmarkDoltgres benchmarks doltgres based on the provided configurations +//func BenchmarkDoltgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { +// serverParams, err := serverConfig.GetServerArgs() +// if err != nil { +// return nil, err +// } +// +// err = DoltVersion(ctx, serverConfig.ServerExec) +// if err != nil { +// return nil, err +// } +// +// serverDir, err := CreateServerDir(dbName) +// if err != nil { +// return nil, err +// } +// defer func() { +// cleanupDoltgresServerDir(serverDir) +// }() +// +// serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir)) +// +// withKeyCtx, cancel := context.WithCancel(ctx) +// gServer, serverCtx := errgroup.WithContext(withKeyCtx) +// +// server := getServer(serverCtx, serverConfig, serverDir, serverParams) +// +// // handle user interrupt +// quit := make(chan os.Signal, 1) +// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) +// var wg sync.WaitGroup +// wg.Add(1) +// go func() { +// <-quit +// defer wg.Done() +// signal.Stop(quit) +// cancel() +// }() +// +// // launch the dolt server +// gServer.Go(func() error { +// return server.Run() +// }) +// +// // sleep to allow the server to start +// time.Sleep(5 * time.Second) +// +// // create the db against the running server +// err = createDb(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "doltgres", dbName) +// if err != nil { +// close(quit) +// wg.Wait() +// return nil, err +// } +// +// tests, err := GetTests(config, serverConfig, nil) +// if err != nil { +// close(quit) +// wg.Wait() +// return nil, err +// } +// +// results := make(Results, 0) +// for i := 0; i < config.Runs; i++ { +// for _, test := range tests { +// r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) +// if err != nil { +// close(quit) +// wg.Wait() +// return nil, err +// } +// results = append(results, r) +// } +// } +// +// // send signal to dolt server +// quit <- syscall.SIGTERM +// +// err = gServer.Wait() +// if err != nil { +// // we expect a kill error +// // we only exit in error if this is not the +// // error +// if err.Error() != "signal: killed" { +// fmt.Println(err) +// close(quit) +// wg.Wait() +// return nil, err +// } +// } +// +// fmt.Println("Successfully killed server") +// close(quit) +// wg.Wait() +// +// return results, nil +//} -// cleanupDoltgresServerDir cleans up the doltgres assets in the provided dir -func cleanupDoltgresServerDir(dir string) error { - dataDir := filepath.Join(dir, ".dolt") - defaultDir := filepath.Join(dir, "doltgres") - testDir := filepath.Join(dir, dbName) - for _, d := range []string{dataDir, defaultDir, testDir} { - if _, err := os.Stat(d); !os.IsNotExist(err) { - err = os.RemoveAll(d) - if err != nil { - return err - } - } - } - return nil -} +//func createDb(ctx context.Context, host, port, user, dbname string) error { +// psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname) +// +// // open database +// db, err := sql.Open("postgres", psqlconn) +// if err != nil { +// return err +// } +// +// // close database +// defer db.Close() +// +// // check db +// err = db.PingContext(ctx) +// if err != nil { +// return err +// } +// +// _, err = db.ExecContext(ctx, fmt.Sprintf("create database %s;", dbname)) +// return err +//} + +//// cleanupDoltgresServerDir cleans up the doltgres assets in the provided dir +//func cleanupDoltgresServerDir(dir string) error { +// dataDir := filepath.Join(dir, ".dolt") +// defaultDir := filepath.Join(dir, "doltgres") +// testDir := filepath.Join(dir, dbName) +// for _, d := range []string{dataDir, defaultDir, testDir} { +// if _, err := os.Stat(d); !os.IsNotExist(err) { +// err = os.RemoveAll(d) +// if err != nil { +// return err +// } +// } +// } +// return nil +//} diff --git a/go/performance/utils/sysbench_runner/mysql.go b/go/performance/utils/sysbench_runner/mysql.go index 82735bf4056..28426d1cf70 100644 --- a/go/performance/utils/sysbench_runner/mysql.go +++ b/go/performance/utils/sysbench_runner/mysql.go @@ -4,10 +4,10 @@ import ( "context" "database/sql" "fmt" - _ "github.com/go-sql-driver/mysql" - "log" "os" "syscall" + + _ "github.com/go-sql-driver/mysql" ) const ( @@ -116,7 +116,6 @@ func (b *mysqlBenchmarkerImpl) createTestingDb(ctx context.Context) (err error) } } - log.Println("Successfully set up the MySQL database") return } diff --git a/go/performance/utils/sysbench_runner/mysql_old.go b/go/performance/utils/sysbench_runner/mysql_old.go index e6f66a791c6..b6dbe1f8fd7 100644 --- a/go/performance/utils/sysbench_runner/mysql_old.go +++ b/go/performance/utils/sysbench_runner/mysql_old.go @@ -18,16 +18,8 @@ import ( "context" "database/sql" "fmt" - "log" - "os" - "os/exec" - "os/signal" - "sync" - "syscall" - "time" _ "github.com/go-sql-driver/mysql" - "golang.org/x/sync/errgroup" ) type MysqlConfig struct { @@ -37,119 +29,119 @@ type MysqlConfig struct { Host string } -// BenchmarkMysql benchmarks mysql based on the provided configurations -func BenchmarkMysql(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - withKeyCtx, cancel := context.WithCancel(ctx) - - var serverDir string - defer func() { - if serverDir != "" { - os.RemoveAll(serverDir) - } - }() - - var localServer bool - var gServer *errgroup.Group - var serverCtx context.Context - var server *exec.Cmd - var err error - if serverConfig.Host == defaultHost { - log.Println("Launching the default server") - localServer = true - - serverDir, err = InitMysqlDataDir(ctx, serverConfig) - if err != nil { - cancel() - return nil, err - } - - gServer, serverCtx = errgroup.WithContext(withKeyCtx) - var serverParams []string - serverParams, err = serverConfig.GetServerArgs() - if err != nil { - cancel() - return nil, err - } - serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) - - server = getMysqlServer(serverCtx, serverConfig, serverParams) - - // launch the mysql server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(10 * time.Second) - - // setup mysqldb - err := SetupDB(ctx, GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName) - if err != nil { - cancel() - return nil, err - } - log.Println("Successfully set up the MySQL database") - } - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - tests, err := GetTests(config, serverConfig, nil) - if err != nil { - return nil, err - } - - results := make(Results, 0) - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - results = append(results, r) - } - } - - // stop local mysql server - if localServer { - // send signal to server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - close(quit) - wg.Wait() - return nil, err - } - } - } - - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - return results, nil -} +//// BenchmarkMysql benchmarks mysql based on the provided configurations +//func BenchmarkMysql(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { +// withKeyCtx, cancel := context.WithCancel(ctx) +// +// var serverDir string +// defer func() { +// if serverDir != "" { +// os.RemoveAll(serverDir) +// } +// }() +// +// var localServer bool +// var gServer *errgroup.Group +// var serverCtx context.Context +// var server *exec.Cmd +// var err error +// if serverConfig.Host == defaultHost { +// log.Println("Launching the default server") +// localServer = true +// +// serverDir, err = InitMysqlDataDir(ctx, serverConfig) +// if err != nil { +// cancel() +// return nil, err +// } +// +// gServer, serverCtx = errgroup.WithContext(withKeyCtx) +// var serverParams []string +// serverParams, err = serverConfig.GetServerArgs() +// if err != nil { +// cancel() +// return nil, err +// } +// serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) +// +// server = getMysqlServer(serverCtx, serverConfig, serverParams) +// +// // launch the mysql server +// gServer.Go(func() error { +// return server.Run() +// }) +// +// // sleep to allow the server to start +// time.Sleep(10 * time.Second) +// +// // setup mysqldb +// err := SetupDB(ctx, GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName) +// if err != nil { +// cancel() +// return nil, err +// } +// log.Println("Successfully set up the MySQL database") +// } +// +// // handle user interrupt +// quit := make(chan os.Signal, 1) +// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) +// var wg sync.WaitGroup +// wg.Add(1) +// go func() { +// <-quit +// defer wg.Done() +// signal.Stop(quit) +// cancel() +// }() +// +// tests, err := GetTests(config, serverConfig, nil) +// if err != nil { +// return nil, err +// } +// +// results := make(Results, 0) +// for i := 0; i < config.Runs; i++ { +// for _, test := range tests { +// r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) +// if err != nil { +// close(quit) +// wg.Wait() +// return nil, err +// } +// results = append(results, r) +// } +// } +// +// // stop local mysql server +// if localServer { +// // send signal to server +// quit <- syscall.SIGTERM +// +// err = gServer.Wait() +// if err != nil { +// // we expect a kill error +// // we only exit in error if this is not the +// // error +// if err.Error() != "signal: killed" { +// close(quit) +// wg.Wait() +// return nil, err +// } +// } +// } +// +// fmt.Println("Successfully killed server") +// close(quit) +// wg.Wait() +// +// return results, nil +//} -// getMysqlServer returns a exec.Cmd for a dolt server -func getMysqlServer(ctx context.Context, config *ServerConfig, params []string) *exec.Cmd { - return ExecCommand(ctx, config.ServerExec, params...) -} +//// getMysqlServer returns a exec.Cmd for a dolt server +//func getMysqlServer(ctx context.Context, config *ServerConfig, params []string) *exec.Cmd { +// return ExecCommand(ctx, config.ServerExec, params...) +//} // InitMysqlDataDir initializes a mysql data dir and returns the path func InitMysqlDataDir(ctx context.Context, config *ServerConfig) (string, error) { diff --git a/go/performance/utils/sysbench_runner/postgres.go b/go/performance/utils/sysbench_runner/postgres.go index 0e2a0323270..9414270254f 100644 --- a/go/performance/utils/sysbench_runner/postgres.go +++ b/go/performance/utils/sysbench_runner/postgres.go @@ -1,160 +1,48 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package sysbench_runner import ( "context" + "database/sql" "fmt" - "log" - "os" - "os/exec" - "os/signal" - "sync" "syscall" - "time" - - "database/sql" _ "github.com/lib/pq" - "golang.org/x/sync/errgroup" ) -type PostgresConfig struct { - Socket string - ConnectionProtocol string - Port int - Host string -} - -// BenchmarkPostgres benchmarks postgres based on the provided configurations -func BenchmarkPostgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - withKeyCtx, cancel := context.WithCancel(ctx) - - var serverDir string - defer func() { - if serverDir != "" { - os.RemoveAll(serverDir) - } - }() - - var localServer bool - var gServer *errgroup.Group - var serverCtx context.Context - var server *exec.Cmd - var err error - if serverConfig.Host == defaultHost { - log.Println("Launching the default server") - localServer = true - - serverDir, err = initPostgresDataDir(ctx, serverConfig) - if err != nil { - cancel() - return nil, err - } - gServer, serverCtx = errgroup.WithContext(withKeyCtx) - var serverParams []string - serverParams, err = serverConfig.GetServerArgs() - if err != nil { - cancel() - return nil, err - } - serverParams = append(serverParams, "-D", serverDir) - server = getMysqlServer(serverCtx, serverConfig, serverParams) - server.Env = append(server.Env, "LC_ALL=C") - - // launch the postgres server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(10 * time.Second) - - // setup postgres db - err := setupPostgresDB(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "postgres", dbName) - if err != nil { - cancel() - return nil, err - } - - log.Println("Successfully set up the Postgres database") - } +const ( + postgresInitDbDataDirFlag = "--pgdata" + postgresUsernameFlag = "--username" + postgresUsername = "postgres" + postgresDataDirFlag = "-D" + postgresDropDatabaseSqlTemplate = "DROP DATABASE IF EXISTS %s;" + postgresDropUserSqlTemplate = "DROP USER IF EXISTS %s;" + postgresCreateUserSqlTemplate = "CREATE USER %s WITH PASSWORD '%s';" + postgresCreateDatabaseSqlTemplate = "CREATE DATABASE %s WITH OWNER %s;" +) - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() +type postgresBenchmarkerImpl struct { + dir string // cwd + config *Config + serverConfig *ServerConfig +} - tests, err := GetTests(config, serverConfig, nil) - if err != nil { - return nil, err - } +var _ Benchmarker = &postgresBenchmarkerImpl{} - results := make(Results, 0) - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - results = append(results, r) - } +func NewPostgresBenchmarker(dir string, config *Config, serverConfig *ServerConfig) *postgresBenchmarkerImpl { + return &postgresBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, } - - // stop local mysql server - if localServer { - // send signal to server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - close(quit) - wg.Wait() - return nil, err - } - } - } - - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - return results, nil } -// initPostgresDataDir initializes a postgres data dir and returns the path -func initPostgresDataDir(ctx context.Context, config *ServerConfig) (string, error) { +func (b *postgresBenchmarkerImpl) initDataDir(ctx context.Context) (string, error) { serverDir, err := CreateServerDir(dbName) if err != nil { return "", err } - pgInit := ExecCommand(ctx, config.InitExec, fmt.Sprintf("--pgdata=%s", serverDir), "--username=postgres") + pgInit := ExecCommand(ctx, b.serverConfig.InitExec, fmt.Sprintf("%s=%s", postgresInitDbDataDirFlag, serverDir), fmt.Sprintf("%s=%s", postgresUsernameFlag, postgresUsername)) err = pgInit.Run() if err != nil { return "", err @@ -163,12 +51,13 @@ func initPostgresDataDir(ctx context.Context, config *ServerConfig) (string, err return serverDir, nil } -func setupPostgresDB(ctx context.Context, host, port, user, dbname string) (err error) { - psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname) +func (b *postgresBenchmarkerImpl) createTestingDb(ctx context.Context) (err error) { + psqlconn := fmt.Sprintf(psqlDsnTemplate, b.serverConfig.Host, b.serverConfig.Port, postgresUsername, "", dbName) - db, err := sql.Open("postgres", psqlconn) + var db *sql.DB + db, err = sql.Open("postgres", psqlconn) if err != nil { - return err + return } defer func() { rerr := db.Close() @@ -178,23 +67,71 @@ func setupPostgresDB(ctx context.Context, host, port, user, dbname string) (err }() err = db.PingContext(ctx) if err != nil { - return err + return + } + + stmts := []string{ + fmt.Sprintf(postgresDropDatabaseSqlTemplate, dbName), + fmt.Sprintf(postgresDropUserSqlTemplate, sysbenchUsername), + fmt.Sprintf(postgresCreateUserSqlTemplate, sysbenchUsername, sysbenchPassLocal), + fmt.Sprintf(postgresCreateDatabaseSqlTemplate, dbName, sysbenchUsername), } - _, err = db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", dbname)) + + for _, s := range stmts { + _, err = db.ExecContext(ctx, s) + if err != nil { + return + } + } + + return +} + +func (b *postgresBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { + serverDir, err := b.initDataDir(ctx) if err != nil { - return err + return nil, err } - _, err = db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s", sysbenchUsername)) + + serverParams, err := b.serverConfig.GetServerArgs() if err != nil { - return err + return nil, err } - _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE USER %s WITH PASSWORD '%s'", sysbenchUsername, sysbenchPassLocal)) + serverParams = append(serverParams, postgresDataDirFlag, serverDir) + + server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start(ctx) if err != nil { - return err + return nil, err } - _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE %s WITH OWNER %s", dbname, sysbenchUsername)) + + err = b.createTestingDb(ctx) if err != nil { - return err + return nil, err } - return + + tests, err := GetTests(b.config, b.serverConfig, nil) + if err != nil { + return nil, err + } + + results := make(Results, 0) + for i := 0; i < b.config.Runs; i++ { + for _, test := range tests { + tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop(ctx) + return nil, err + } + results = append(results, r) + } + } + + err = server.Stop(ctx) + if err != nil { + return nil, err + } + + return results, nil } diff --git a/go/performance/utils/sysbench_runner/postgres_old.go b/go/performance/utils/sysbench_runner/postgres_old.go new file mode 100644 index 00000000000..10fcec03b29 --- /dev/null +++ b/go/performance/utils/sysbench_runner/postgres_old.go @@ -0,0 +1,187 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +//import ( +// _ "github.com/lib/pq" +//) + +//type PostgresConfig struct { +// Socket string +// ConnectionProtocol string +// Port int +// Host string +//} + +//// BenchmarkPostgres benchmarks postgres based on the provided configurations +//func BenchmarkPostgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { +// withKeyCtx, cancel := context.WithCancel(ctx) +// +// var serverDir string +// defer func() { +// if serverDir != "" { +// os.RemoveAll(serverDir) +// } +// }() +// +// var localServer bool +// var gServer *errgroup.Group +// var serverCtx context.Context +// var server *exec.Cmd +// var err error +// if serverConfig.Host == defaultHost { +// log.Println("Launching the default server") +// localServer = true +// +// serverDir, err = initPostgresDataDir(ctx, serverConfig) +// if err != nil { +// cancel() +// return nil, err +// } +// gServer, serverCtx = errgroup.WithContext(withKeyCtx) +// var serverParams []string +// serverParams, err = serverConfig.GetServerArgs() +// if err != nil { +// cancel() +// return nil, err +// } +// serverParams = append(serverParams, "-D", serverDir) +// server = getMysqlServer(serverCtx, serverConfig, serverParams) +// server.Env = append(server.Env, "LC_ALL=C") +// +// // launch the postgres server +// gServer.Go(func() error { +// return server.Run() +// }) +// +// // sleep to allow the server to start +// time.Sleep(10 * time.Second) +// +// // setup postgres db +// err := setupPostgresDB(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "postgres", dbName) +// if err != nil { +// cancel() +// return nil, err +// } +// +// log.Println("Successfully set up the Postgres database") +// } +// +// // handle user interrupt +// quit := make(chan os.Signal, 1) +// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) +// var wg sync.WaitGroup +// wg.Add(1) +// go func() { +// <-quit +// defer wg.Done() +// signal.Stop(quit) +// cancel() +// }() +// +// tests, err := GetTests(config, serverConfig, nil) +// if err != nil { +// return nil, err +// } +// +// results := make(Results, 0) +// for i := 0; i < config.Runs; i++ { +// for _, test := range tests { +// r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) +// if err != nil { +// close(quit) +// wg.Wait() +// return nil, err +// } +// results = append(results, r) +// } +// } +// +// // stop local mysql server +// if localServer { +// // send signal to server +// quit <- syscall.SIGTERM +// +// err = gServer.Wait() +// if err != nil { +// // we expect a kill error +// // we only exit in error if this is not the +// // error +// if err.Error() != "signal: killed" { +// close(quit) +// wg.Wait() +// return nil, err +// } +// } +// } +// +// fmt.Println("Successfully killed server") +// close(quit) +// wg.Wait() +// +// return results, nil +//} + +//// initPostgresDataDir initializes a postgres data dir and returns the path +//func initPostgresDataDir(ctx context.Context, config *ServerConfig) (string, error) { +// serverDir, err := CreateServerDir(dbName) +// if err != nil { +// return "", err +// } +// +// pgInit := ExecCommand(ctx, config.InitExec, fmt.Sprintf("--pgdata=%s", serverDir), "--username=postgres") +// err = pgInit.Run() +// if err != nil { +// return "", err +// } +// +// return serverDir, nil +//} + +//func setupPostgresDB(ctx context.Context, host, port, user, dbname string) (err error) { +// psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname) +// +// db, err := sql.Open("postgres", psqlconn) +// if err != nil { +// return err +// } +// defer func() { +// rerr := db.Close() +// if err == nil { +// err = rerr +// } +// }() +// err = db.PingContext(ctx) +// if err != nil { +// return err +// } +// _, err = db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", dbname)) +// if err != nil { +// return err +// } +// _, err = db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s", sysbenchUsername)) +// if err != nil { +// return err +// } +// _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE USER %s WITH PASSWORD '%s'", sysbenchUsername, sysbenchPassLocal)) +// if err != nil { +// return err +// } +// _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE %s WITH OWNER %s", dbname, sysbenchUsername)) +// if err != nil { +// return err +// } +// return +//} diff --git a/go/performance/utils/sysbench_runner/run.go b/go/performance/utils/sysbench_runner/run.go index 45010dbddb3..d8917b92371 100644 --- a/go/performance/utils/sysbench_runner/run.go +++ b/go/performance/utils/sysbench_runner/run.go @@ -42,6 +42,7 @@ func Run(config *Config) error { for _, serverConfig := range config.Servers { var results Results + var b Benchmarker switch serverConfig.Server { case Dolt: // handle a profiling run @@ -49,26 +50,22 @@ func Run(config *Config) error { fmt.Println("Profiling dolt while running sysbench tests") return ProfileDolt(ctx, config, serverConfig) } - fmt.Println("Running dolt sysbench test") - //results, err = BenchmarkDolt(ctx, config, serverConfig) - b := NewDoltBenchmarker(cwd, config, serverConfig) - results, err = b.Benchmark(ctx) + fmt.Println("Running dolt sysbench tests") + b = NewDoltBenchmarker(cwd, config, serverConfig) case Doltgres: - fmt.Println("Running doltgres sysbench test") - b := NewDoltgresBenchmarker(cwd, config, serverConfig) - results, err = b.Benchmark(ctx) - //results, err = BenchmarkDoltgres(ctx, config, serverConfig) + fmt.Println("Running doltgres sysbench tests") + b = NewDoltgresBenchmarker(cwd, config, serverConfig) case MySql: - fmt.Println("Running mysql sysbench test") - b := NewMysqlBenchmarker(cwd, config, serverConfig) - results, err = b.Benchmark(ctx) - //results, err = BenchmarkMysql(ctx, config, serverConfig) + fmt.Println("Running mysql sysbench tests") + b = NewMysqlBenchmarker(cwd, config, serverConfig) case Postgres: - fmt.Println("Running postgres sysbench test") - results, err = BenchmarkPostgres(ctx, config, serverConfig) + fmt.Println("Running postgres sysbench tests") + b = NewPostgresBenchmarker(cwd, config, serverConfig) default: panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server)) } + + results, err = b.Benchmark(ctx) if err != nil { return err } diff --git a/go/performance/utils/sysbench_runner/server.go b/go/performance/utils/sysbench_runner/server.go index dac67a972db..832aa5495cf 100644 --- a/go/performance/utils/sysbench_runner/server.go +++ b/go/performance/utils/sysbench_runner/server.go @@ -2,15 +2,17 @@ package sysbench_runner import ( "context" + "errors" "fmt" - "golang.org/x/sync/errgroup" "os" "os/exec" - "os/signal" - "sync" "time" + + "golang.org/x/sync/errgroup" ) +var ErrServerClosed = errors.New("server was previously closed") + type Server interface { Start(ctx context.Context) error Stop(ctx context.Context) error @@ -24,7 +26,6 @@ type doltServerImpl struct { server *exec.Cmd serverEg *errgroup.Group quit chan os.Signal - wg *sync.WaitGroup killSignal os.Signal } @@ -41,37 +42,39 @@ func NewServer(ctx context.Context, dir string, serverConfig *ServerConfig, kill return &doltServerImpl{ dir: dir, serverConfig: serverConfig, + serverCtx: serverCtx, server: server, serverCtxCancelFunc: cancel, serverEg: gServer, quit: quit, - wg: &sync.WaitGroup{}, killSignal: killSignal, } } func (s *doltServerImpl) Start(ctx context.Context) error { - signal.Notify(s.quit, os.Interrupt, s.killSignal) - s.wg.Add(1) - go func() { + if s.serverEg == nil || s.serverCtx == nil || s.quit == nil { + return ErrServerClosed + } + + s.serverEg.Go(func() error { <-s.quit - defer s.wg.Done() - signal.Stop(s.quit) - // todo: remove this ?? s.serverCtxCancelFunc() - }() + return s.server.Process.Signal(s.killSignal) + }) // launch the dolt server s.serverEg.Go(func() error { return s.server.Run() }) + // sleep to allow the server to start time.Sleep(10 * time.Second) + fmt.Println("Successfully started database server") return nil } func (s *doltServerImpl) Stop(ctx context.Context) error { defer s.serverCtxCancelFunc() - if s.wg != nil && s.serverEg != nil && s.serverCtx != nil && s.quit != nil { + if s.serverEg != nil && s.serverCtx != nil && s.quit != nil { // send signal to dolt server s.quit <- s.killSignal err := s.serverEg.Wait() @@ -82,16 +85,13 @@ func (s *doltServerImpl) Stop(ctx context.Context) error { if err.Error() != "signal: killed" { fmt.Println(err) close(s.quit) - s.wg.Wait() return err } } - fmt.Println("Successfully killed server") + fmt.Println("Successfully killed database server") close(s.quit) - s.wg.Wait() - s.wg = nil s.quit = nil s.serverCtx = nil s.serverEg = nil diff --git a/go/performance/utils/sysbench_runner/sysbench.go b/go/performance/utils/sysbench_runner/sysbench.go index cc225b077e4..e4e432d1e82 100644 --- a/go/performance/utils/sysbench_runner/sysbench.go +++ b/go/performance/utils/sysbench_runner/sysbench.go @@ -3,16 +3,18 @@ package sysbench_runner import ( "context" "fmt" - "github.com/google/uuid" "os" "os/exec" "path/filepath" "strings" + + "github.com/google/uuid" ) const ( sysbenchCommand = "sysbench" luaPathEnvVarTemplate = "LUA_PATH=%s" + luaPath = "?.lua" ) type SysbenchTester interface { @@ -22,7 +24,6 @@ type SysbenchTester interface { type sysbenchTestImpl struct { test *Test config *Config - scriptDir string serverConfig *ServerConfig stampFunc func() string idFunc func() string @@ -125,7 +126,7 @@ func (t *sysbenchTestImpl) outputToResult(output []byte) (*Result, error) { func (t *sysbenchTestImpl) prepare(ctx context.Context) error { cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.Prepare()...) if t.test.FromScript { - lp := filepath.Join(t.scriptDir, luaPath) + lp := filepath.Join(t.config.ScriptDir, luaPath) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) } @@ -140,7 +141,7 @@ func (t *sysbenchTestImpl) prepare(ctx context.Context) error { func (t *sysbenchTestImpl) run(ctx context.Context) (*Result, error) { cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.Run()...) if t.test.FromScript { - lp := filepath.Join(t.scriptDir, luaPath) + lp := filepath.Join(t.config.ScriptDir, luaPath) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) } @@ -168,7 +169,7 @@ func (t *sysbenchTestImpl) run(ctx context.Context) (*Result, error) { func (t *sysbenchTestImpl) cleanup(ctx context.Context) error { cmd := ExecCommand(ctx, sysbenchCommand, t.test.Cleanup()...) if t.test.FromScript { - lp := filepath.Join(t.scriptDir, luaPath) + lp := filepath.Join(t.config.ScriptDir, luaPath) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) } @@ -181,7 +182,7 @@ func (t *sysbenchTestImpl) Test(ctx context.Context) (*Result, error) { return nil, err } - fmt.Println("Running test ", t.test.Name) + fmt.Println("Running test", t.test.Name) rs, err := t.run(ctx) if err != nil { From 09268b4475c838836f28fba069bcd9350a33284c Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 12 Feb 2024 12:21:21 -0800 Subject: [PATCH 06/25] /go/performance/utils/sysbench_runner: wip, servers seem to work, now fixing profiling, then can refactor the rest --- go/performance/utils/sysbench_runner/dolt.go | 6 +-- .../utils/sysbench_runner/dolt_old.go | 3 +- .../utils/sysbench_runner/doltgres.go | 48 ++++++++++------- go/performance/utils/sysbench_runner/mysql.go | 6 +-- .../utils/sysbench_runner/postgres.go | 52 ++++++++++++------- .../utils/sysbench_runner/server.go | 27 +++++++--- 6 files changed, 91 insertions(+), 51 deletions(-) diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go index 620efb6fd6f..95818728e48 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -130,7 +130,7 @@ func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { } server := NewServer(ctx, testRepo, b.serverConfig, syscall.SIGTERM, serverParams) - err = server.Start(ctx) + err = server.Start() if err != nil { return nil, err } @@ -146,14 +146,14 @@ func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) r, err := tester.Test(ctx) if err != nil { - server.Stop(ctx) + server.Stop() return nil, err } results = append(results, r) } } - err = server.Stop(ctx) + err = server.Stop() if err != nil { return nil, err } diff --git a/go/performance/utils/sysbench_runner/dolt_old.go b/go/performance/utils/sysbench_runner/dolt_old.go index deafc05daa7..bf48627ca12 100644 --- a/go/performance/utils/sysbench_runner/dolt_old.go +++ b/go/performance/utils/sysbench_runner/dolt_old.go @@ -17,10 +17,11 @@ package sysbench_runner import ( "context" "fmt" - "github.com/dolthub/dolt/go/store/types" "os" "os/exec" "path/filepath" + + "github.com/dolthub/dolt/go/store/types" ) const ( diff --git a/go/performance/utils/sysbench_runner/doltgres.go b/go/performance/utils/sysbench_runner/doltgres.go index 7d1e1ecc8f5..6713b1ca30e 100644 --- a/go/performance/utils/sysbench_runner/doltgres.go +++ b/go/performance/utils/sysbench_runner/doltgres.go @@ -79,61 +79,69 @@ func (b *doltgresBenchmarkerImpl) createTestingDb(ctx context.Context) error { return err } -func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { - err := b.checkInstallation(ctx) +func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Results, err error) { + err = b.checkInstallation(ctx) if err != nil { - return nil, err + return } - serverDir, err := CreateServerDir(dbName) + var serverDir string + serverDir, err = CreateServerDir(dbName) if err != nil { - return nil, err + return } defer func() { - b.cleanupServerDir(serverDir) + rerr := b.cleanupServerDir(serverDir) + if err == nil { + err = rerr + } }() - serverParams, err := b.serverConfig.GetServerArgs() + var serverParams []string + serverParams, err = b.serverConfig.GetServerArgs() if err != nil { - return nil, err + return } + serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir)) server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) - err = server.Start(ctx) + err = server.Start() if err != nil { - return nil, err + return } err = b.createTestingDb(ctx) if err != nil { - return nil, err + return } - tests, err := GetTests(b.config, b.serverConfig, nil) + var tests []*Test + tests, err = GetTests(b.config, b.serverConfig, nil) if err != nil { - return nil, err + return } - results := make(Results, 0) + results = make(Results, 0) for i := 0; i < b.config.Runs; i++ { for _, test := range tests { tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) - r, err := tester.Test(ctx) + var r *Result + r, err = tester.Test(ctx) if err != nil { - server.Stop(ctx) - return nil, err + server.Stop() + return } results = append(results, r) } } - err = server.Stop(ctx) + err = server.Stop() if err != nil { - return nil, err + return } - return results, nil + return } // CreateServerDir creates a server directory diff --git a/go/performance/utils/sysbench_runner/mysql.go b/go/performance/utils/sysbench_runner/mysql.go index 28426d1cf70..2ca86337b8b 100644 --- a/go/performance/utils/sysbench_runner/mysql.go +++ b/go/performance/utils/sysbench_runner/mysql.go @@ -132,7 +132,7 @@ func (b *mysqlBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) - err = server.Start(ctx) + err = server.Start() if err != nil { return nil, err } @@ -153,14 +153,14 @@ func (b *mysqlBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) r, err := tester.Test(ctx) if err != nil { - server.Stop(ctx) + server.Stop() return nil, err } results = append(results, r) } } - err = server.Stop(ctx) + err = server.Stop() if err != nil { return nil, err } diff --git a/go/performance/utils/sysbench_runner/postgres.go b/go/performance/utils/sysbench_runner/postgres.go index 9414270254f..7024969d5f1 100644 --- a/go/performance/utils/sysbench_runner/postgres.go +++ b/go/performance/utils/sysbench_runner/postgres.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "os" "syscall" _ "github.com/lib/pq" @@ -18,6 +19,8 @@ const ( postgresDropUserSqlTemplate = "DROP USER IF EXISTS %s;" postgresCreateUserSqlTemplate = "CREATE USER %s WITH PASSWORD '%s';" postgresCreateDatabaseSqlTemplate = "CREATE DATABASE %s WITH OWNER %s;" + postgresLcAllEnvVarKey = "LC_ALL" + postgresLcAllEnvVarValue = "C" ) type postgresBenchmarkerImpl struct { @@ -55,7 +58,7 @@ func (b *postgresBenchmarkerImpl) createTestingDb(ctx context.Context) (err erro psqlconn := fmt.Sprintf(psqlDsnTemplate, b.serverConfig.Host, b.serverConfig.Port, postgresUsername, "", dbName) var db *sql.DB - db, err = sql.Open("postgres", psqlconn) + db, err = sql.Open(postgresDriver, psqlconn) if err != nil { return } @@ -87,51 +90,64 @@ func (b *postgresBenchmarkerImpl) createTestingDb(ctx context.Context) (err erro return } -func (b *postgresBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { - serverDir, err := b.initDataDir(ctx) +func (b *postgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Results, err error) { + var serverDir string + serverDir, err = b.initDataDir(ctx) if err != nil { - return nil, err + return } + defer func() { + rerr := os.RemoveAll(serverDir) + if err == nil { + err = rerr + } + }() - serverParams, err := b.serverConfig.GetServerArgs() + var serverParams []string + serverParams, err = b.serverConfig.GetServerArgs() if err != nil { - return nil, err + return } + serverParams = append(serverParams, postgresDataDirFlag, serverDir) server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) - err = server.Start(ctx) + server.WithEnv(postgresLcAllEnvVarKey, postgresLcAllEnvVarValue) + + err = server.Start() if err != nil { - return nil, err + return } err = b.createTestingDb(ctx) if err != nil { - return nil, err + return } - tests, err := GetTests(b.config, b.serverConfig, nil) + var tests []*Test + tests, err = GetTests(b.config, b.serverConfig, nil) if err != nil { - return nil, err + return } - results := make(Results, 0) + results = make(Results, 0) for i := 0; i < b.config.Runs; i++ { for _, test := range tests { tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) - r, err := tester.Test(ctx) + var r *Result + r, err = tester.Test(ctx) if err != nil { - server.Stop(ctx) - return nil, err + server.Stop() + return } results = append(results, r) } } - err = server.Stop(ctx) + err = server.Stop() if err != nil { - return nil, err + return } - return results, nil + return } diff --git a/go/performance/utils/sysbench_runner/server.go b/go/performance/utils/sysbench_runner/server.go index 832aa5495cf..4439c8164bc 100644 --- a/go/performance/utils/sysbench_runner/server.go +++ b/go/performance/utils/sysbench_runner/server.go @@ -11,11 +11,17 @@ import ( "golang.org/x/sync/errgroup" ) +const ( + expectedServerKilledErrorMessage = "signal: killed" + expectedServerTerminatedErrorMessage = "signal: terminated" +) + var ErrServerClosed = errors.New("server was previously closed") type Server interface { - Start(ctx context.Context) error - Stop(ctx context.Context) error + Start() error + Stop() error + WithEnv(key, value string) } type doltServerImpl struct { @@ -51,7 +57,13 @@ func NewServer(ctx context.Context, dir string, serverConfig *ServerConfig, kill } } -func (s *doltServerImpl) Start(ctx context.Context) error { +func (s *doltServerImpl) WithEnv(key, val string) { + if s.server != nil { + s.server.Env = append(s.server.Env, fmt.Sprintf("%s=%s", key, val)) + } +} + +func (s *doltServerImpl) Start() error { if s.serverEg == nil || s.serverCtx == nil || s.quit == nil { return ErrServerClosed } @@ -61,8 +73,11 @@ func (s *doltServerImpl) Start(ctx context.Context) error { return s.server.Process.Signal(s.killSignal) }) - // launch the dolt server s.serverEg.Go(func() error { + if Debug { + s.server.Stdout = os.Stdout + s.server.Stderr = os.Stderr + } return s.server.Run() }) @@ -72,7 +87,7 @@ func (s *doltServerImpl) Start(ctx context.Context) error { return nil } -func (s *doltServerImpl) Stop(ctx context.Context) error { +func (s *doltServerImpl) Stop() error { defer s.serverCtxCancelFunc() if s.serverEg != nil && s.serverCtx != nil && s.quit != nil { // send signal to dolt server @@ -82,7 +97,7 @@ func (s *doltServerImpl) Stop(ctx context.Context) error { // we expect a kill error // we only exit in error if this is not the // error - if err.Error() != "signal: killed" { + if err.Error() != expectedServerKilledErrorMessage && err.Error() != expectedServerTerminatedErrorMessage { fmt.Println(err) close(s.quit) return err From 1162d2c4c5dc6eff08ec0b287d6dfe3f63e43b4d Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 12 Feb 2024 13:29:07 -0800 Subject: [PATCH 07/25] /go/performance/utils/sysbench_runner: wip, profiling seems to work --- go/performance/utils/sysbench_runner/dolt.go | 105 ++++----- .../utils/sysbench_runner/profile.go | 207 ++++++------------ .../utils/sysbench_runner/profile_old.go | 169 ++++++++++++++ go/performance/utils/sysbench_runner/run.go | 3 +- 4 files changed, 287 insertions(+), 197 deletions(-) create mode 100644 go/performance/utils/sysbench_runner/profile_old.go diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go index 95818728e48..0ed4486b3a3 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -45,30 +45,12 @@ func NewDoltBenchmarker(dir string, config *Config, serverConfig *ServerConfig) } } -// checkSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary -func (b *doltBenchmarkerImpl) checkSetDoltConfig(ctx context.Context, key, val string) error { - check := ExecCommand(ctx, b.serverConfig.ServerExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigGetFlag, key) - err := check.Run() - if err != nil { - // config get calls exit with 1 if not set - if err.Error() != "exit status 1" { - return err - } - set := ExecCommand(ctx, b.serverConfig.ServerExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigAddFlag, key, val) - err := set.Run() - if err != nil { - return err - } - } - return nil -} - func (b *doltBenchmarkerImpl) updateGlobalConfig(ctx context.Context) error { - err := b.checkSetDoltConfig(ctx, doltConfigUsernameKey, doltBenchmarkUser) + err := CheckSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigUsernameKey, doltBenchmarkUser) if err != nil { return err } - return b.checkSetDoltConfig(ctx, doltConfigEmailKey, doltBenchmarkEmail) + return CheckSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigEmailKey, doltBenchmarkEmail) } func (b *doltBenchmarkerImpl) checkInstallation(ctx context.Context) error { @@ -76,36 +58,8 @@ func (b *doltBenchmarkerImpl) checkInstallation(ctx context.Context) error { return version.Run() } -// initDoltRepo initializes a dolt repo and returns the repo path func (b *doltBenchmarkerImpl) initDoltRepo(ctx context.Context) (string, error) { - testRepo := filepath.Join(b.dir, dbName) - if b.config.NomsBinFormat == types.Format_LD_1.VersionString() { - err := ExecCommand(ctx, b.serverConfig.ServerExec, doltCloneCommand, bigEmptyRepo, dbName).Run() - if err != nil { - return "", err - } - return testRepo, nil - } - - err := os.MkdirAll(testRepo, os.ModePerm) - if err != nil { - return "", err - } - - if b.config.NomsBinFormat != "" { - if err = os.Setenv(nbfEnvVar, b.config.NomsBinFormat); err != nil { - return "", err - } - } - - doltInit := ExecCommand(ctx, b.serverConfig.ServerExec, doltInitCommand) - doltInit.Dir = testRepo - err = doltInit.Run() - if err != nil { - return "", err - } - - return testRepo, nil + return InitDoltRepo(ctx, b.dir, b.serverConfig.ServerExec, b.config.NomsBinFormat) } func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { @@ -123,6 +77,7 @@ func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { if err != nil { return nil, err } + defer os.RemoveAll(testRepo) serverParams, err := b.serverConfig.GetServerArgs() if err != nil { @@ -158,5 +113,55 @@ func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { return nil, err } - return results, os.RemoveAll(testRepo) + return results, nil +} + +// InitDoltRepo initializes a dolt database and returns its path +func InitDoltRepo(ctx context.Context, dir, serverExec, nomsBinFormat string) (string, error) { + testRepo := filepath.Join(dir, dbName) + if nomsBinFormat == types.Format_LD_1.VersionString() { + err := ExecCommand(ctx, serverExec, doltCloneCommand, bigEmptyRepo, dbName).Run() + if err != nil { + return "", err + } + return testRepo, nil + } + + err := os.MkdirAll(testRepo, os.ModePerm) + if err != nil { + return "", err + } + + if nomsBinFormat != "" { + if err = os.Setenv(nbfEnvVar, nomsBinFormat); err != nil { + return "", err + } + } + + doltInit := ExecCommand(ctx, serverExec, doltInitCommand) + doltInit.Dir = testRepo + err = doltInit.Run() + if err != nil { + return "", err + } + + return testRepo, nil +} + +// CheckSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary +func CheckSetDoltConfig(ctx context.Context, serverExec, key, val string) error { + check := ExecCommand(ctx, serverExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigGetFlag, key) + err := check.Run() + if err != nil { + // config get calls exit with 1 if not set + if err.Error() != "exit status 1" { + return err + } + set := ExecCommand(ctx, serverExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigAddFlag, key, val) + err := set.Run() + if err != nil { + return err + } + } + return nil } diff --git a/go/performance/utils/sysbench_runner/profile.go b/go/performance/utils/sysbench_runner/profile.go index feb27873e9b..f6b15e0200b 100644 --- a/go/performance/utils/sysbench_runner/profile.go +++ b/go/performance/utils/sysbench_runner/profile.go @@ -1,94 +1,75 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package sysbench_runner import ( "context" "fmt" - "io" "os" - "os/signal" "path/filepath" - "sync" "syscall" - "time" - - "golang.org/x/sync/errgroup" ) -// ProfileDolt profiles dolt while running the provided tests -func ProfileDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) error { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return err - } +type Profiler interface { + Profile(ctx context.Context) error +} - err = DoltVersion(ctx, serverConfig.ServerExec) - if err != nil { - return err - } +type doltProfilerImpl struct { + dir string // cwd + config *Config + serverConfig *ServerConfig +} - err = UpdateDoltConfig(ctx, serverConfig.ServerExec) - if err != nil { - return err +var _ Profiler = &doltProfilerImpl{} + +func NewDoltProfiler(dir string, config *Config, serverConfig *ServerConfig) *doltProfilerImpl { + return &doltProfilerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, } +} - testRepo, err := initDoltRepo(ctx, serverConfig, config.NomsBinFormat) +func (p *doltProfilerImpl) updateGlobalConfig(ctx context.Context) error { + err := CheckSetDoltConfig(ctx, p.serverConfig.ServerExec, doltConfigUsernameKey, doltBenchmarkUser) if err != nil { return err } + return CheckSetDoltConfig(ctx, p.serverConfig.ServerExec, doltConfigEmailKey, doltBenchmarkEmail) +} - tests, err := GetTests(config, serverConfig, nil) +func (p *doltProfilerImpl) checkInstallation(ctx context.Context) error { + version := ExecCommand(ctx, p.serverConfig.ServerExec, doltVersionCommand) + return version.Run() +} + +func (p *doltProfilerImpl) initDoltRepo(ctx context.Context) (string, error) { + return InitDoltRepo(ctx, p.dir, p.serverConfig.ServerExec, p.config.NomsBinFormat) +} + +func (p *doltProfilerImpl) Profile(ctx context.Context) error { + err := p.checkInstallation(ctx) if err != nil { return err } - tempProfilesDir, err := os.MkdirTemp("", "") + err = p.updateGlobalConfig(ctx) if err != nil { return err } - defer os.RemoveAll(tempProfilesDir) - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - _, err = profileTest(ctx, test, config, serverConfig, serverParams, testRepo, tempProfilesDir) - if err != nil { - return err - } - } - } - - profile, err := mergeProfiles(ctx, tempProfilesDir, serverConfig.ProfilePath) + testRepo, err := p.initDoltRepo(ctx) if err != nil { return err } + defer os.RemoveAll(testRepo) - fmt.Println("Profile created at:", profile) - - err = os.RemoveAll(testRepo) + serverParams, err := p.serverConfig.GetServerArgs() if err != nil { return err } - return nil -} - -func profileTest(ctx context.Context, test *Test, config *Config, serverConfig *ServerConfig, serverParams []string, testRepo, profileDir string) (string, error) { - profilePath, err := os.MkdirTemp("", filepath.Base(test.Name)) + profilePath, err := os.MkdirTemp("", "dolt_profile_path_*") if err != nil { - return "", err + return err } defer os.RemoveAll(profilePath) @@ -97,110 +78,44 @@ func profileTest(ctx context.Context, test *Test, config *Config, serverConfig * profileParams = append(profileParams, profileFlag, cpuProfile, profilePathFlag, profilePath) profileParams = append(profileParams, serverParams...) - withKeyCtx, cancel := context.WithCancel(ctx) - defer cancel() - gServer, serverCtx := errgroup.WithContext(withKeyCtx) - server := getServer(serverCtx, serverConfig, testRepo, profileParams) - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGINT) - var wg sync.WaitGroup - wg.Add(1) - go func() { - s := <-quit - defer wg.Done() - server.Process.Signal(s) - signal.Stop(quit) - }() - - // launch the dolt server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(5 * time.Second) - - _, err = benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) + server := NewServer(ctx, testRepo, p.serverConfig, syscall.SIGTERM, profileParams) + err = server.Start() if err != nil { - close(quit) - wg.Wait() - return "", err - } - - // send signal to dolt server - quit <- syscall.SIGINT - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - fmt.Println(err) - close(quit) - wg.Wait() - return "", err - } - } - - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - info, err := os.Stat(tempProfile) - if err != nil { - return "", err - } - if info.Size() < 1 { - return "", fmt.Errorf("failed to create profile: file was empty") + return err } - finalProfile := filepath.Join(profileDir, fmt.Sprintf("%s_%s_%s", serverConfig.Id, test.Name, cpuProfileFilename)) - err = moveFile(tempProfile, finalProfile) - return finalProfile, err -} - -func mergeProfiles(ctx context.Context, sourceProfilesDir, destProfileDir string) (string, error) { - tmp, err := os.MkdirTemp("", "final_cpu_pprof") + tests, err := GetTests(p.config, p.serverConfig, nil) if err != nil { - return "", err + return err } - defer os.RemoveAll(tmp) - outfile := filepath.Join(tmp, "cpu.pprof") - merge := ExecCommand(ctx, "/bin/sh", "-c", fmt.Sprintf("go tool pprof -proto *.pprof > %s", outfile)) - merge.Dir = sourceProfilesDir - err = merge.Run() - if err != nil { - return "", err + results := make(Results, 0) + for i := 0; i < p.config.Runs; i++ { + for _, test := range tests { + tester := NewSysbenchTester(p.config, p.serverConfig, test, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop() + return err + } + results = append(results, r) + } } - final := filepath.Join(destProfileDir, "cpu.pprof") - err = moveFile(outfile, final) - return final, err -} - -func moveFile(sourcePath, destPath string) error { - err := copyFile(sourcePath, destPath) + err = server.Stop() if err != nil { return err } - return os.Remove(sourcePath) -} -func copyFile(sourcePath, destPath string) error { - inputFile, err := os.Open(sourcePath) + info, err := os.Stat(tempProfile) if err != nil { return err } - defer inputFile.Close() - outputFile, err := os.Create(destPath) - if err != nil { - return err + + if info.Size() < 1 { + return fmt.Errorf("failed to create profile: file was empty") } - defer outputFile.Close() - _, err = io.Copy(outputFile, inputFile) - return err + + finalProfile := filepath.Join(p.serverConfig.ProfilePath, fmt.Sprintf("%s_%s", p.serverConfig.Id, cpuProfileFilename)) + return os.Rename(tempProfile, finalProfile) } diff --git a/go/performance/utils/sysbench_runner/profile_old.go b/go/performance/utils/sysbench_runner/profile_old.go new file mode 100644 index 00000000000..00867711ec4 --- /dev/null +++ b/go/performance/utils/sysbench_runner/profile_old.go @@ -0,0 +1,169 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +//// ProfileDolt profiles dolt while running the provided tests +//func ProfileDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) error { +// serverParams, err := serverConfig.GetServerArgs() +// if err != nil { +// return err +// } +// +// err = DoltVersion(ctx, serverConfig.ServerExec) +// if err != nil { +// return err +// } +// +// err = UpdateDoltConfig(ctx, serverConfig.ServerExec) +// if err != nil { +// return err +// } +// +// testRepo, err := initDoltRepo(ctx, serverConfig, config.NomsBinFormat) +// if err != nil { +// return err +// } +// +// tests, err := GetTests(config, serverConfig, nil) +// if err != nil { +// return err +// } +// +// tempProfilesDir, err := os.MkdirTemp("", "") +// if err != nil { +// return err +// } +// defer os.RemoveAll(tempProfilesDir) +// +// for i := 0; i < config.Runs; i++ { +// for _, test := range tests { +// _, err = profileTest(ctx, test, config, serverConfig, serverParams, testRepo, tempProfilesDir) +// if err != nil { +// return err +// } +// } +// } +// +// profile, err := mergeProfiles(ctx, tempProfilesDir, serverConfig.ProfilePath) +// if err != nil { +// return err +// } +// +// fmt.Println("Profile created at:", profile) +// +// err = os.RemoveAll(testRepo) +// if err != nil { +// return err +// } +// +// return nil +//} + +//func profileTest(ctx context.Context, test *Test, config *Config, serverConfig *ServerConfig, serverParams []string, testRepo, profileDir string) (string, error) { +// profilePath, err := os.MkdirTemp("", filepath.Base(test.Name)) +// if err != nil { +// return "", err +// } +// defer os.RemoveAll(profilePath) +// +// tempProfile := filepath.Join(profilePath, cpuProfileFilename) +// profileParams := make([]string, 0) +// profileParams = append(profileParams, profileFlag, cpuProfile, profilePathFlag, profilePath) +// profileParams = append(profileParams, serverParams...) +// +// withKeyCtx, cancel := context.WithCancel(ctx) +// defer cancel() +// gServer, serverCtx := errgroup.WithContext(withKeyCtx) +// server := getServer(serverCtx, serverConfig, testRepo, profileParams) +// +// // handle user interrupt +// quit := make(chan os.Signal, 1) +// signal.Notify(quit, os.Interrupt, syscall.SIGINT) +// var wg sync.WaitGroup +// wg.Add(1) +// go func() { +// s := <-quit +// defer wg.Done() +// server.Process.Signal(s) +// signal.Stop(quit) +// }() +// +// // launch the dolt server +// gServer.Go(func() error { +// return server.Run() +// }) +// +// // sleep to allow the server to start +// time.Sleep(5 * time.Second) +// +// _, err = benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) +// if err != nil { +// close(quit) +// wg.Wait() +// return "", err +// } +// +// // send signal to dolt server +// quit <- syscall.SIGINT +// +// err = gServer.Wait() +// if err != nil { +// // we expect a kill error +// // we only exit in error if this is not the +// // error +// if err.Error() != "signal: killed" { +// fmt.Println(err) +// close(quit) +// wg.Wait() +// return "", err +// } +// } +// +// fmt.Println("Successfully killed server") +// close(quit) +// wg.Wait() +// +// info, err := os.Stat(tempProfile) +// if err != nil { +// return "", err +// } +// if info.Size() < 1 { +// return "", fmt.Errorf("failed to create profile: file was empty") +// } +// +// finalProfile := filepath.Join(profileDir, fmt.Sprintf("%s_%s_%s", serverConfig.Id, test.Name, cpuProfileFilename)) +// err = moveFile(tempProfile, finalProfile) +// return finalProfile, err +//} + +//func mergeProfiles(ctx context.Context, sourceProfilesDir, destProfileDir string) (string, error) { +// tmp, err := os.MkdirTemp("", "final_cpu_pprof") +// if err != nil { +// return "", err +// } +// defer os.RemoveAll(tmp) +// outfile := filepath.Join(tmp, "cpu.pprof") +// +// merge := ExecCommand(ctx, "/bin/sh", "-c", fmt.Sprintf("go tool pprof -proto *.pprof > %s", outfile)) +// merge.Dir = sourceProfilesDir +// err = merge.Run() +// if err != nil { +// return "", err +// } +// +// final := filepath.Join(destProfileDir, "cpu.pprof") +// err = moveFile(outfile, final) +// return final, err +//} diff --git a/go/performance/utils/sysbench_runner/run.go b/go/performance/utils/sysbench_runner/run.go index d8917b92371..c56324ef711 100644 --- a/go/performance/utils/sysbench_runner/run.go +++ b/go/performance/utils/sysbench_runner/run.go @@ -48,7 +48,8 @@ func Run(config *Config) error { // handle a profiling run if serverConfig.ServerProfile != "" { fmt.Println("Profiling dolt while running sysbench tests") - return ProfileDolt(ctx, config, serverConfig) + p := NewDoltProfiler(cwd, config, serverConfig) + return p.Profile(ctx) } fmt.Println("Running dolt sysbench tests") b = NewDoltBenchmarker(cwd, config, serverConfig) From 5e066a5c93ab9aa8ee4e386c5d161a9f868b4cbd Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 12 Feb 2024 14:48:35 -0800 Subject: [PATCH 08/25] /go/performance/utils/{sysbench_runner,tpcc_runner}: wip, refactoring tpcc stuff --- go/performance/utils/sysbench_runner/dolt.go | 6 +- .../utils/sysbench_runner/dolt_tpcc.go | 107 ++++++++ go/performance/utils/sysbench_runner/mysql.go | 142 +++++----- .../utils/sysbench_runner/mysql_old.go | 30 +-- .../utils/sysbench_runner/mysql_tpcc.go | 80 ++++++ .../utils/sysbench_runner/results.go | 58 +++++ .../utils/sysbench_runner/sysbench.go | 51 +--- .../utils/sysbench_runner/tester.go | 7 + go/performance/utils/sysbench_runner/tpcc.go | 86 ++++++ .../utils/sysbench_runner/tpcc_config.go | 246 ++++++++++++++++++ go/performance/utils/tpcc_runner/mysql.go | 2 +- 11 files changed, 680 insertions(+), 135 deletions(-) create mode 100644 go/performance/utils/sysbench_runner/dolt_tpcc.go create mode 100644 go/performance/utils/sysbench_runner/mysql_tpcc.go create mode 100644 go/performance/utils/sysbench_runner/tester.go create mode 100644 go/performance/utils/sysbench_runner/tpcc.go create mode 100644 go/performance/utils/sysbench_runner/tpcc_config.go diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go index 0ed4486b3a3..30430416243 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -59,7 +59,7 @@ func (b *doltBenchmarkerImpl) checkInstallation(ctx context.Context) error { } func (b *doltBenchmarkerImpl) initDoltRepo(ctx context.Context) (string, error) { - return InitDoltRepo(ctx, b.dir, b.serverConfig.ServerExec, b.config.NomsBinFormat) + return InitDoltRepo(ctx, b.dir, b.serverConfig.ServerExec, b.config.NomsBinFormat, dbName) } func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { @@ -98,7 +98,7 @@ func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { results := make(Results, 0) for i := 0; i < b.config.Runs; i++ { for _, test := range tests { - tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) + tester := NewSysbenchTester(b.config, b.serverConfig, test, serverParams, stampFunc) r, err := tester.Test(ctx) if err != nil { server.Stop() @@ -117,7 +117,7 @@ func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { } // InitDoltRepo initializes a dolt database and returns its path -func InitDoltRepo(ctx context.Context, dir, serverExec, nomsBinFormat string) (string, error) { +func InitDoltRepo(ctx context.Context, dir, serverExec, nomsBinFormat, dbName string) (string, error) { testRepo := filepath.Join(dir, dbName) if nomsBinFormat == types.Format_LD_1.VersionString() { err := ExecCommand(ctx, serverExec, doltCloneCommand, bigEmptyRepo, dbName).Run() diff --git a/go/performance/utils/sysbench_runner/dolt_tpcc.go b/go/performance/utils/sysbench_runner/dolt_tpcc.go new file mode 100644 index 00000000000..92a5a07ac4b --- /dev/null +++ b/go/performance/utils/sysbench_runner/dolt_tpcc.go @@ -0,0 +1,107 @@ +package sysbench_runner + +import ( + "context" + "fmt" + "os" + "syscall" +) + +const ( + tpccDbName = "sbt" + tpccScaleFactorTemplate = "tpcc-scale-factor-%d" +) + +type doltTpccBenchmarkerImpl struct { + dir string // cwd + config *TpccBenchmarkConfig + serverConfig *ServerConfig +} + +var _ Benchmarker = &doltTpccBenchmarkerImpl{} + +func NewDoltTpccBenchmarker(dir string, config *TpccBenchmarkConfig, serverConfig *ServerConfig) *doltTpccBenchmarkerImpl { + return &doltTpccBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, + } +} + +func (b *doltTpccBenchmarkerImpl) updateGlobalConfig(ctx context.Context) error { + err := CheckSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigUsernameKey, doltBenchmarkUser) + if err != nil { + return err + } + return CheckSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigEmailKey, doltBenchmarkEmail) +} + +func (b *doltTpccBenchmarkerImpl) checkInstallation(ctx context.Context) error { + version := ExecCommand(ctx, b.serverConfig.ServerExec, doltVersionCommand) + return version.Run() +} + +func (b *doltTpccBenchmarkerImpl) initDoltRepo(ctx context.Context) (string, error) { + return InitDoltRepo(ctx, b.dir, b.serverConfig.ServerExec, b.config.NomsBinFormat, tpccDbName) +} + +func (b *doltTpccBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { + err := b.checkInstallation(ctx) + if err != nil { + return nil, err + } + + err = b.updateGlobalConfig(ctx) + if err != nil { + return nil, err + } + + testRepo, err := b.initDoltRepo(ctx) + if err != nil { + return nil, err + } + defer os.RemoveAll(testRepo) + + serverParams, err := b.serverConfig.GetServerArgs() + if err != nil { + return nil, err + } + + server := NewServer(ctx, testRepo, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start() + if err != nil { + return nil, err + } + + tests := GetTpccTests(b.config) + + results := make(Results, 0) + for _, test := range tests { + tester := NewTpccTester(b.config, b.serverConfig, test, serverParams, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop() + return nil, err + } + results = append(results, r) + } + + err = server.Stop() + if err != nil { + return nil, err + } + + return results, nil +} + +// GetTpccTests creates a set of tests that the server needs to be executed on. +func GetTpccTests(config *TpccBenchmarkConfig) []*TpccTest { + tests := make([]*TpccTest, 0) + for _, sf := range config.ScaleFactors { + params := NewDefaultTpccParams() + params.ScaleFactor = sf + test := NewTpccTest(fmt.Sprintf(tpccScaleFactorTemplate, sf), params) + tests = append(tests, test) + } + return tests +} diff --git a/go/performance/utils/sysbench_runner/mysql.go b/go/performance/utils/sysbench_runner/mysql.go index 2ca86337b8b..05d3f64a218 100644 --- a/go/performance/utils/sysbench_runner/mysql.go +++ b/go/performance/utils/sysbench_runner/mysql.go @@ -39,49 +39,83 @@ func NewMysqlBenchmarker(dir string, config *Config, serverConfig *ServerConfig) } } -func (b *mysqlBenchmarkerImpl) createServerDir() (string, error) { - return CreateServerDir(dbName) +func (b *mysqlBenchmarkerImpl) getDsn() (string, error) { + return GetMysqlDsn(b.serverConfig.Host, b.serverConfig.Socket, b.serverConfig.ConnectionProtocol, b.serverConfig.Port) } -func (b *mysqlBenchmarkerImpl) initMysqlDataDir(ctx context.Context) (string, error) { - serverDir, err := b.createServerDir() +func (b *mysqlBenchmarkerImpl) createTestingDb(ctx context.Context) error { + dsn, err := b.getDsn() if err != nil { - return "", err + return err } + return CreateMysqlTestingDb(ctx, dsn, dbName) +} - msInit := ExecCommand(ctx, b.serverConfig.ServerExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) - err = msInit.Run() +func (b *mysqlBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { + serverDir, err := InitMysqlDataDir(ctx, b.serverConfig.ServerExec, dbName) if err != nil { - return "", err + return nil, err } - return serverDir, nil -} + serverParams, err := b.serverConfig.GetServerArgs() + if err != nil { + return nil, err + } + serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) -func (b *mysqlBenchmarkerImpl) getDsn() (string, error) { - var socketPath string - if b.serverConfig.Socket != "" { - socketPath = b.serverConfig.Socket - } else { - socketPath = defaultMysqlSocket + server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start() + if err != nil { + return nil, err } - if b.serverConfig.ConnectionProtocol == tcpProtocol { - return fmt.Sprintf(mysqlRootTCPDsnTemplate, b.serverConfig.Host, b.serverConfig.Port), nil - } else if b.serverConfig.ConnectionProtocol == unixProtocol { - return fmt.Sprintf(mysqlRootUnixDsnTemplate, socketPath), nil - } else { - return "", ErrUnsupportedConnectionProtocol + err = b.createTestingDb(ctx) + if err != nil { + return nil, err } + + tests, err := GetTests(b.config, b.serverConfig, nil) + if err != nil { + return nil, err + } + + results := make(Results, 0) + for i := 0; i < b.config.Runs; i++ { + for _, test := range tests { + tester := NewSysbenchTester(b.config, b.serverConfig, test, serverParams, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop() + return nil, err + } + results = append(results, r) + } + } + + err = server.Stop() + if err != nil { + return nil, err + } + + return results, os.RemoveAll(serverDir) } -func (b *mysqlBenchmarkerImpl) createTestingDb(ctx context.Context) (err error) { - var dsn string - dsn, err = b.getDsn() +func InitMysqlDataDir(ctx context.Context, serverExec, dbName string) (string, error) { + serverDir, err := CreateServerDir(dbName) if err != nil { - return + return "", err + } + + msInit := ExecCommand(ctx, serverExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) + err = msInit.Run() + if err != nil { + return "", err } + return serverDir, nil +} + +func CreateMysqlTestingDb(ctx context.Context, dsn, dbName string) (err error) { var db *sql.DB db, err = sql.Open(mysqlDriverName, dsn) if err != nil { @@ -119,51 +153,19 @@ func (b *mysqlBenchmarkerImpl) createTestingDb(ctx context.Context) (err error) return } -func (b *mysqlBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { - serverDir, err := b.initMysqlDataDir(ctx) - if err != nil { - return nil, err - } - - serverParams, err := b.serverConfig.GetServerArgs() - if err != nil { - return nil, err - } - serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) - - server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) - err = server.Start() - if err != nil { - return nil, err - } - - err = b.createTestingDb(ctx) - if err != nil { - return nil, err - } - - tests, err := GetTests(b.config, b.serverConfig, nil) - if err != nil { - return nil, err - } - - results := make(Results, 0) - for i := 0; i < b.config.Runs; i++ { - for _, test := range tests { - tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) - r, err := tester.Test(ctx) - if err != nil { - server.Stop() - return nil, err - } - results = append(results, r) - } +func GetMysqlDsn(host, socket, protocol string, port int) (string, error) { + var socketPath string + if socket != "" { + socketPath = socket + } else { + socketPath = defaultMysqlSocket } - err = server.Stop() - if err != nil { - return nil, err + if protocol == tcpProtocol { + return fmt.Sprintf(mysqlRootTCPDsnTemplate, host, port), nil + } else if protocol == unixProtocol { + return fmt.Sprintf(mysqlRootUnixDsnTemplate, socketPath), nil + } else { + return "", ErrUnsupportedConnectionProtocol } - - return results, os.RemoveAll(serverDir) } diff --git a/go/performance/utils/sysbench_runner/mysql_old.go b/go/performance/utils/sysbench_runner/mysql_old.go index b6dbe1f8fd7..4d456afa2a0 100644 --- a/go/performance/utils/sysbench_runner/mysql_old.go +++ b/go/performance/utils/sysbench_runner/mysql_old.go @@ -143,21 +143,21 @@ type MysqlConfig struct { // return ExecCommand(ctx, config.ServerExec, params...) //} -// InitMysqlDataDir initializes a mysql data dir and returns the path -func InitMysqlDataDir(ctx context.Context, config *ServerConfig) (string, error) { - serverDir, err := CreateServerDir(dbName) - if err != nil { - return "", err - } - - msInit := ExecCommand(ctx, config.ServerExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) - err = msInit.Run() - if err != nil { - return "", err - } - - return serverDir, nil -} +//// InitMysqlDataDir initializes a mysql data dir and returns the path +//func InitMysqlDataDir(ctx context.Context, config *ServerConfig) (string, error) { +// serverDir, err := CreateServerDir(dbName) +// if err != nil { +// return "", err +// } +// +// msInit := ExecCommand(ctx, config.ServerExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) +// err = msInit.Run() +// if err != nil { +// return "", err +// } +// +// return serverDir, nil +//} func SetupDB(ctx context.Context, mConfig MysqlConfig, databaseName string) (err error) { dsn, err := FormatDsn(mConfig) diff --git a/go/performance/utils/sysbench_runner/mysql_tpcc.go b/go/performance/utils/sysbench_runner/mysql_tpcc.go new file mode 100644 index 00000000000..61ff4232e22 --- /dev/null +++ b/go/performance/utils/sysbench_runner/mysql_tpcc.go @@ -0,0 +1,80 @@ +package sysbench_runner + +import ( + "context" + "fmt" + "os" + "syscall" +) + +type mysqlTpccBenchmarkerImpl struct { + dir string // cwd + config *TpccBenchmarkConfig + serverConfig *ServerConfig +} + +var _ Benchmarker = &mysqlTpccBenchmarkerImpl{} + +func NewMysqlTpccBenchmarker(dir string, config *TpccBenchmarkConfig, serverConfig *ServerConfig) *mysqlTpccBenchmarkerImpl { + return &mysqlTpccBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, + } +} + +func (b *mysqlTpccBenchmarkerImpl) getDsn() (string, error) { + return GetMysqlDsn(b.serverConfig.Host, b.serverConfig.Socket, b.serverConfig.ConnectionProtocol, b.serverConfig.Port) +} + +func (b *mysqlTpccBenchmarkerImpl) createTestingDb(ctx context.Context) error { + dsn, err := b.getDsn() + if err != nil { + return err + } + return CreateMysqlTestingDb(ctx, dsn, tpccDbName) +} + +func (b *mysqlTpccBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { + serverDir, err := InitMysqlDataDir(ctx, b.serverConfig.ServerExec, tpccDbName) + if err != nil { + return nil, err + } + + serverParams, err := b.serverConfig.GetServerArgs() + if err != nil { + return nil, err + } + serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) + + server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start() + if err != nil { + return nil, err + } + + err = b.createTestingDb(ctx) + if err != nil { + return nil, err + } + + tests := GetTpccTests(b.config) + + results := make(Results, 0) + for _, test := range tests { + tester := NewTpccTester(b.config, b.serverConfig, test, serverParams, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop() + return nil, err + } + results = append(results, r) + } + + err = server.Stop() + if err != nil { + return nil, err + } + + return results, os.RemoveAll(serverDir) +} diff --git a/go/performance/utils/sysbench_runner/results.go b/go/performance/utils/sysbench_runner/results.go index 5ad7256196b..bb30d1662f0 100644 --- a/go/performance/utils/sysbench_runner/results.go +++ b/go/performance/utils/sysbench_runner/results.go @@ -191,6 +191,39 @@ func FromConfigsNewResult(config *Config, serverConfig *ServerConfig, t *Test, s }, nil } +func NewResult(server ServerType, version, testName, testId, suiteId, runtimeOs, runtimeGoArch string, serverParams, testParams []string, idFunc func() string, fromScript bool) *Result { + var getId func() string + if idFunc == nil { + getId = func() string { + return uuid.New().String() + } + } else { + getId = idFunc + } + + var name string + if fromScript { + base := filepath.Base(testName) + ext := filepath.Ext(base) + name = strings.TrimSuffix(base, ext) + } else { + name = testName + } + + return &Result{ + Id: getId(), + SuiteId: suiteId, + TestId: testId, + RuntimeOS: runtimeOs, + RuntimeGoArch: runtimeGoArch, + ServerName: string(server), + ServerVersion: version, + ServerParams: strings.Join(serverParams, " "), + TestName: name, + TestParams: strings.Join(testParams, " "), + } +} + // FromOutputResult accepts raw sysbench run output and returns the Result func FromOutputResult(output []byte, config *Config, serverConfig *ServerConfig, test *Test, suiteId string, idFunc func() string) (*Result, error) { result, err := FromConfigsNewResult(config, serverConfig, test, suiteId, idFunc) @@ -368,6 +401,31 @@ func updateResult(result *Result, key, val string) error { return nil } +func OutputToResult(output []byte, server ServerType, version, testName, testId, suiteId, runtimeOs, runtimeGoArch string, serverParams, testParams []string, idFunc func() string, fromScript bool) (*Result, error) { + result := NewResult(server, version, testName, testId, suiteId, runtimeOs, runtimeGoArch, serverParams, testParams, idFunc, fromScript) + + lines := strings.Split(string(output), "\n") + var process bool + for _, l := range lines { + trimmed := strings.TrimSpace(l) + if trimmed == "" { + continue + } + if strings.HasPrefix(trimmed, SqlStatsPrefix) { + process = true + continue + } + if process { + err := UpdateResult(result, trimmed) + if err != nil { + return result, err + } + } + } + + return result, nil +} + // FromValWithParens takes a string containing parens and // returns the value outside the parens first, and the value // inside the parens second diff --git a/go/performance/utils/sysbench_runner/sysbench.go b/go/performance/utils/sysbench_runner/sysbench.go index e4e432d1e82..00be936a8eb 100644 --- a/go/performance/utils/sysbench_runner/sysbench.go +++ b/go/performance/utils/sysbench_runner/sysbench.go @@ -17,24 +17,22 @@ const ( luaPath = "?.lua" ) -type SysbenchTester interface { - Test(ctx context.Context) (*Result, error) -} - type sysbenchTestImpl struct { test *Test config *Config serverConfig *ServerConfig + serverParams []string stampFunc func() string idFunc func() string suiteId string } -var _ SysbenchTester = &sysbenchTestImpl{} +var _ Tester = &sysbenchTestImpl{} -func NewSysbenchTester(config *Config, serverConfig *ServerConfig, test *Test, stampFunc func() string) *sysbenchTestImpl { +func NewSysbenchTester(config *Config, serverConfig *ServerConfig, test *Test, serverParams []string, stampFunc func() string) *sysbenchTestImpl { return &sysbenchTestImpl{ config: config, + serverParams: serverParams, serverConfig: serverConfig, test: test, suiteId: serverConfig.GetId(), @@ -80,47 +78,8 @@ func (t *sysbenchTestImpl) newResult() (*Result, error) { }, nil } -func (t *sysbenchTestImpl) updateResult(result *Result, line string) error { - lineParts := strings.Split(line, ":") - key := strings.TrimSpace(lineParts[0]) - - if len(lineParts) > 1 { - rawVal := strings.TrimSpace(lineParts[1]) - err := updateResult(result, key, rawVal) - if err != nil { - return err - } - } - - return nil -} - func (t *sysbenchTestImpl) outputToResult(output []byte) (*Result, error) { - result, err := t.newResult() - if err != nil { - return nil, err - } - - lines := strings.Split(string(output), "\n") - var process bool - for _, l := range lines { - trimmed := strings.TrimSpace(l) - if trimmed == "" { - continue - } - if strings.HasPrefix(trimmed, SqlStatsPrefix) { - process = true - continue - } - if process { - err := t.updateResult(result, trimmed) - if err != nil { - return result, err - } - } - } - - return result, nil + return OutputToResult(output, t.serverConfig.Server, t.serverConfig.Version, t.test.Name, t.test.id, t.suiteId, t.config.RuntimeOS, t.config.RuntimeGoArch, t.serverParams, t.test.Params, nil, t.test.FromScript) } func (t *sysbenchTestImpl) prepare(ctx context.Context) error { diff --git a/go/performance/utils/sysbench_runner/tester.go b/go/performance/utils/sysbench_runner/tester.go new file mode 100644 index 00000000000..68bf22dac76 --- /dev/null +++ b/go/performance/utils/sysbench_runner/tester.go @@ -0,0 +1,7 @@ +package sysbench_runner + +import "context" + +type Tester interface { + Test(ctx context.Context) (*Result, error) +} diff --git a/go/performance/utils/sysbench_runner/tpcc.go b/go/performance/utils/sysbench_runner/tpcc.go new file mode 100644 index 00000000000..edec15b623a --- /dev/null +++ b/go/performance/utils/sysbench_runner/tpcc.go @@ -0,0 +1,86 @@ +package sysbench_runner + +import ( + "context" + "fmt" +) + +type tpccTestImpl struct { + test *TpccTest + config *TpccBenchmarkConfig + serverConfig *ServerConfig + serverParams []string + stampFunc func() string + idFunc func() string + suiteId string +} + +var _ Tester = &tpccTestImpl{} + +func NewTpccTester(config *TpccBenchmarkConfig, serverConfig *ServerConfig, test *TpccTest, serverParams []string, stampFunc func() string) *tpccTestImpl { + return &tpccTestImpl{ + config: config, + serverParams: serverParams, + serverConfig: serverConfig, + test: test, + suiteId: serverConfig.GetId(), + stampFunc: stampFunc, + } +} + +func (t *tpccTestImpl) outputToResult(output []byte) (*Result, error) { + return OutputToResult(output, t.serverConfig.Server, t.serverConfig.Version, t.test.Name, t.test.Id, t.suiteId, t.config.RuntimeOS, t.config.RuntimeGoArch, t.serverParams, t.test.Params.ToSlice(), nil, false) +} + +func (t *tpccTestImpl) prepare(ctx context.Context) error { + cmd := t.test.TpccPrepare(ctx, t.serverConfig, t.config.ScriptDir) + out, err := cmd.Output() + if err != nil { + fmt.Println(string(out)) + return err + } + return nil +} + +func (t *tpccTestImpl) run(ctx context.Context) (*Result, error) { + cmd := t.test.TpccRun(ctx, t.serverConfig, t.config.ScriptDir) + out, err := cmd.Output() + if err != nil { + fmt.Print(string(out)) + return nil, err + } + + if Debug == true { + fmt.Print(string(out)) + } + + rs, err := t.outputToResult(out) + if err != nil { + return nil, err + } + + rs.Stamp(t.stampFunc) + + return rs, nil +} + +func (t *tpccTestImpl) cleanup(ctx context.Context) error { + cmd := t.test.TpccCleanup(ctx, t.serverConfig, t.config.ScriptDir) + return cmd.Run() +} + +func (t *tpccTestImpl) Test(ctx context.Context) (*Result, error) { + err := t.prepare(ctx) + if err != nil { + return nil, err + } + + fmt.Println("Running test", t.test.Name) + + rs, err := t.run(ctx) + if err != nil { + return nil, err + } + + return rs, t.cleanup(ctx) +} diff --git a/go/performance/utils/sysbench_runner/tpcc_config.go b/go/performance/utils/sysbench_runner/tpcc_config.go new file mode 100644 index 00000000000..5cf2d79cd92 --- /dev/null +++ b/go/performance/utils/sysbench_runner/tpcc_config.go @@ -0,0 +1,246 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + + "github.com/google/uuid" +) + +const ( + defaultUser = "root" + + // Note this is built for the SysbenchDocker file. If you want to run locally you'll need to override these variables + // for your local MySQL setup. + tpccUserLocal = "'sysbench'@'localhost'" + tpccPassLocal = "sysbenchpass" +) + +var defaultTpccParams = []string{ + fmt.Sprintf("--mysql-db=%s", dbName), + "--db-driver=mysql", +} + +// TpccBenchmarkConfig represents a configuration for an execution of the TPCC Benchmark. It executes a series of tests +// against different ServerConfigurations. +type TpccBenchmarkConfig struct { + // RuntimeOS is the platform the benchmarks ran on + RuntimeOS string + + // RuntimeGoArch is the runtime architecture + RuntimeGoArch string + + // ScriptDir represents the location of the TPCC tests + ScriptDir string + + // Servers are the servers to benchmark. + Servers []*ServerConfig + + // ScaleFactors represent the scale at which to run each TpccBenchmark at. + ScaleFactors []int + + // NomsBinFormat specifies the NomsBinFormat + NomsBinFormat string +} + +func NewTpccConfig() *TpccBenchmarkConfig { + return &TpccBenchmarkConfig{ + Servers: make([]*ServerConfig, 0), + ScaleFactors: make([]int, 0), + } +} + +func (c *TpccBenchmarkConfig) updateDefaults() error { + if len(c.Servers) < 1 { + return ErrNoServersDefined + } + + // TODO: Eventually we need to support scale factors all the way to 10 + if len(c.ScaleFactors) == 0 { + c.ScaleFactors = append(c.ScaleFactors, 1) + } + + if c.RuntimeOS == "" { + c.RuntimeOS = runtime.GOOS + } + if c.RuntimeGoArch == "" { + c.RuntimeGoArch = runtime.GOARCH + } + + return c.validateServerConfigs() +} + +// validateServerConfigs ensures the ServerConfigs are valid +func (c *TpccBenchmarkConfig) validateServerConfigs() error { + portMap := make(map[int]ServerType) + for _, s := range c.Servers { + if s.Server != Dolt && s.Server != MySql { + return fmt.Errorf("unsupported server type: %s", s.Server) + } + + err := ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat) + if err != nil { + return err + } + + if s.Server == MySql { + err = CheckProtocol(s.ConnectionProtocol) + if err != nil { + return err + } + } + + if s.Host == "" { + s.Host = defaultHost + } + + portMap, err = CheckUpdatePortMap(s, portMap) + if err != nil { + return err + } + + err = CheckExec(s.ServerExec, "server exec") + if err != nil { + return err + } + } + return nil +} + +// TpccTest encapsulates an End to End prepare, run, cleanup test case. +type TpccTest struct { + // Id represents a unique test id + Id string + + // Name represents the name of the test case + Name string + + // Params are associated parameters this test runs with + Params *TpccTestParams +} + +type TpccTestParams struct { + // NumThreads represents the number of threads running queries concurrently. + NumThreads int + + // ScaleFactor represents the number of warehouse to test this at scale. + ScaleFactor int + + // Tables represents the number of tables created per warehouse. + Tables int + + // TrxLevel represents what transaction level to use + TrxLevel string + + // ReportCSV determines whether to report output as a csv. + ReportCSV bool + + // ReportInterval defines how often the tpcc benchmark outputs performance stats. + ReportInterval int + + // Time represents how long + Time int +} + +func (t *TpccTestParams) ToSlice() []string { + params := make([]string, 0) + params = append(params, fmt.Sprintf("numThreads=%d", t.NumThreads)) + params = append(params, fmt.Sprintf("scaleFactor=%d", t.ScaleFactor)) + params = append(params, fmt.Sprintf("tables=%d", t.Tables)) + params = append(params, fmt.Sprintf("trxLevel=%s", t.TrxLevel)) + params = append(params, fmt.Sprintf("reportCsv=%t", t.ReportCSV)) + params = append(params, fmt.Sprintf("reportInterval=%d", t.ReportInterval)) + params = append(params, fmt.Sprintf("time=%d", t.Time)) + return params +} + +// NewDefaultTpccParams returns default TpccTestParams. +func NewDefaultTpccParams() *TpccTestParams { + return &TpccTestParams{ + NumThreads: 2, // TODO: When ready, expose as command line argument. + ScaleFactor: 1, + Tables: 1, + TrxLevel: "RR", + ReportCSV: true, + ReportInterval: 1, + Time: 30, + } +} + +// NewTpccTest instantiates and returns a TPCC test. +func NewTpccTest(name string, params *TpccTestParams) *TpccTest { + return &TpccTest{ + Id: uuid.New().String(), + Name: name, + Params: params, + } +} + +// getArgs returns a test's args for all TPCC steps +func (t *TpccTest) getArgs(serverConfig *ServerConfig) []string { + params := make([]string, 0) + params = append(params, defaultTpccParams...) + + params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host)) + + // handle sysbench user for local mysql server + if serverConfig.Server == MySql && serverConfig.Host == defaultHost { + params = append(params, fmt.Sprintf("--mysql-user=%s", "sysbench")) + params = append(params, fmt.Sprintf("--mysql-password=%s", tpccPassLocal)) + } else { + params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port)) + params = append(params, fmt.Sprintf("--mysql-user=%s", defaultUser)) + } + + params = append(params, fmt.Sprintf("--time=%d", t.Params.Time)) + params = append(params, fmt.Sprintf("--threads=%d", t.Params.NumThreads)) + params = append(params, fmt.Sprintf("--report_interval=%d", t.Params.ReportInterval)) + params = append(params, fmt.Sprintf("--tables=%d", t.Params.Tables)) + params = append(params, fmt.Sprintf("--scale=%d", t.Params.ScaleFactor)) + params = append(params, fmt.Sprintf("--trx_level=%s", t.Params.TrxLevel)) + + return params +} + +// TpccPrepare prepares the command executable for the Prepare step. +func (t *TpccTest) TpccPrepare(ctx context.Context, serverConfig *ServerConfig, scriptDir string) *exec.Cmd { + cmd := ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "prepare")...) + return addParamsToCmd(cmd, scriptDir) +} + +// TpccRun prepares the command executable for the Run step. +func (t *TpccTest) TpccRun(ctx context.Context, serverConfig *ServerConfig, scriptDir string) *exec.Cmd { + cmd := exec.CommandContext(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "run")...) + return addParamsToCmd(cmd, scriptDir) +} + +// TpccCleanup prepares the cleanup executable for the Cleanup step. +func (t *TpccTest) TpccCleanup(ctx context.Context, serverConfig *ServerConfig, scriptDir string) *exec.Cmd { + cmd := ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "cleanup")...) + return addParamsToCmd(cmd, scriptDir) +} + +func addParamsToCmd(cmd *exec.Cmd, scriptDir string) *exec.Cmd { + lp := filepath.Join(scriptDir, "?.lua") + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) + return cmd +} diff --git a/go/performance/utils/tpcc_runner/mysql.go b/go/performance/utils/tpcc_runner/mysql.go index 19a4e593a40..ab4b33874b0 100644 --- a/go/performance/utils/tpcc_runner/mysql.go +++ b/go/performance/utils/tpcc_runner/mysql.go @@ -47,7 +47,7 @@ func BenchmarkMysql(ctx context.Context, config *TpccBenchmarkConfig, serverConf var err error if serverConfig.Host == defaultHost { localServer = true - serverDir, err = sysbench_runner.InitMysqlDataDir(ctx, serverConfig) + serverDir, err = sysbench_runner.InitMysqlDataDir(ctx, serverConfig.ServerExec) if err != nil { cancel() return nil, err From 985238b423b59731785453bf0b4f264e17aa98c5 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 12 Feb 2024 15:08:18 -0800 Subject: [PATCH 09/25] /go/performance/utils/sysbench_runner: wip, refactoring tpcc, needs testing --- .../utils/sysbench_runner/cmd/main.go | 24 +++++-- go/performance/utils/sysbench_runner/run.go | 4 +- .../utils/sysbench_runner/run_tpcc.go | 62 +++++++++++++++++++ .../utils/sysbench_runner/tpcc_config.go | 17 +++++ 4 files changed, 100 insertions(+), 7 deletions(-) create mode 100644 go/performance/utils/sysbench_runner/run_tpcc.go diff --git a/go/performance/utils/sysbench_runner/cmd/main.go b/go/performance/utils/sysbench_runner/cmd/main.go index 19bdb28d3fe..07143dc3613 100644 --- a/go/performance/utils/sysbench_runner/cmd/main.go +++ b/go/performance/utils/sysbench_runner/cmd/main.go @@ -15,6 +15,7 @@ package main import ( + "context" "flag" "log" "os" @@ -23,6 +24,7 @@ import ( runner "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" ) +var tpcc = flag.Bool("tpcc", false, "run tpcc benchmarks") var configFile = flag.String("config", "", "path to config file") func main() { @@ -40,15 +42,29 @@ func main() { log.Fatal(err) } - config, err := runner.FromFileConfig(configPath) + ctx := context.Background() + if *tpcc { + err = runTpcc(ctx, configPath) + } else { + err = run(ctx, configPath) + } if err != nil { log.Fatal(err) } +} - err = runner.Run(config) +func runTpcc(ctx context.Context, configPath string) error { + config, err := runner.FromFileTpccConfig(configPath) if err != nil { - log.Fatal(err) + return err } + return runner.RunTpcc(ctx, config) +} - os.Exit(0) +func run(ctx context.Context, configPath string) error { + config, err := runner.FromFileConfig(configPath) + if err != nil { + return err + } + return runner.Run(ctx, config) } diff --git a/go/performance/utils/sysbench_runner/run.go b/go/performance/utils/sysbench_runner/run.go index c56324ef711..2d5dfd58142 100644 --- a/go/performance/utils/sysbench_runner/run.go +++ b/go/performance/utils/sysbench_runner/run.go @@ -22,14 +22,12 @@ import ( ) // Run runs sysbench runner -func Run(config *Config) error { +func Run(ctx context.Context, config *Config) error { err := config.Validate() if err != nil { return err } - ctx := context.Background() - err = sysbenchVersion(ctx) if err != nil { return err diff --git a/go/performance/utils/sysbench_runner/run_tpcc.go b/go/performance/utils/sysbench_runner/run_tpcc.go new file mode 100644 index 00000000000..96f0c168b0f --- /dev/null +++ b/go/performance/utils/sysbench_runner/run_tpcc.go @@ -0,0 +1,62 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "context" + "fmt" + "os" +) + +func RunTpcc(ctx context.Context, config *TpccBenchmarkConfig) error { + err := config.updateDefaults() + if err != nil { + return err + } + + cwd, err := os.Getwd() + if err != nil { + return err + } + + for _, serverConfig := range config.Servers { + var b Benchmarker + var results Results + switch serverConfig.Server { + case Dolt: + fmt.Println("Running dolt tpcc benchmarks") + b = NewDoltTpccBenchmarker(cwd, config, serverConfig) + case MySql: + fmt.Println("Running mysql tpcc benchmarks") + b = NewMysqlTpccBenchmarker(cwd, config, serverConfig) + default: + panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server)) + } + + results, err = b.Benchmark(ctx) + if err != nil { + return err + } + + err = WriteResults(serverConfig, results) + if err != nil { + return err + } + + fmt.Printf("Successfuly wrote results for %s\n", serverConfig.Server) + } + + return nil +} diff --git a/go/performance/utils/sysbench_runner/tpcc_config.go b/go/performance/utils/sysbench_runner/tpcc_config.go index 5cf2d79cd92..1dced53895f 100644 --- a/go/performance/utils/sysbench_runner/tpcc_config.go +++ b/go/performance/utils/sysbench_runner/tpcc_config.go @@ -16,6 +16,7 @@ package sysbench_runner import ( "context" + "encoding/json" "fmt" "os" "os/exec" @@ -244,3 +245,19 @@ func addParamsToCmd(cmd *exec.Cmd, scriptDir string) *exec.Cmd { cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) return cmd } + +// FromFileTpccConfig returns a validated Config based on the config file at the configPath +func FromFileTpccConfig(configPath string) (*TpccBenchmarkConfig, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return nil, err + } + + config := NewTpccConfig() + err = json.Unmarshal(data, config) + if err != nil { + return nil, err + } + + return config, nil +} From 6065eab29c9eee41355e6fca6a4be80e51f4f11a Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 12 Feb 2024 15:30:45 -0800 Subject: [PATCH 10/25] /go/performance/utils/{sysbench_runner,tpcc_runner}: wip, tpcc seems to work still --- .../utils/sysbench_runner/README.md | 36 ++ .../utils/sysbench_runner/doltgres.go | 2 +- .../utils/sysbench_runner/postgres.go | 2 +- .../utils/sysbench_runner/profile.go | 4 +- .../utils/sysbench_runner/tpcc_config.go | 2 +- go/performance/utils/tpcc_runner/cmd/main.go | 86 +-- go/performance/utils/tpcc_runner/config.go | 504 +++++++++--------- go/performance/utils/tpcc_runner/dolt.go | 360 ++++++------- go/performance/utils/tpcc_runner/mysql.go | 238 ++++----- go/performance/utils/tpcc_runner/results.go | 140 ++--- go/performance/utils/tpcc_runner/run.go | 89 ++-- .../utils/tpcc_runner/sample-tpcc-config.json | 6 +- 12 files changed, 741 insertions(+), 728 deletions(-) diff --git a/go/performance/utils/sysbench_runner/README.md b/go/performance/utils/sysbench_runner/README.md index 73aa8192cee..f5a02c9dc8c 100644 --- a/go/performance/utils/sysbench_runner/README.md +++ b/go/performance/utils/sysbench_runner/README.md @@ -99,3 +99,39 @@ oltp_update_non_index `sysbench [options]... [testname] [command]` Note: Be sure that all mysql processes are off when running this locally. + +# TPCC + +TPCC runner is a tool for running TPCC tests against sql servers. These tests run against the +Percona Labs repo [here](https://github.com/Percona-Lab/sysbench-tpcc). + +The tool requires a json config file to run. + +```bash +$ go run cmd/main.go --config=sample-tpcc-config.json +``` + +Note to this run this locally you need to have the TPCC repo cloned. The `ScriptDir` variable should then be linked +to the path of the cloned repo. + +Configuration: + +```json +{ + "Servers": "[...]", + "ScriptDir":"/Users/vinairachakonda/go/src/dolthub/sysbench-tpcc", + "ScaleFactors": [1] +} +``` + +`Servers`: The server defintions to run the benchmark against. Accepts Dolt and MySQL configuratiosn. + +`ScriptDir`: The directory of the TPCC testing scripts + +`ScaleFactors`: The number of warehouse to be generated in the test case. + +`NomsBinFormat`: The NomsBinFormat to use for this benchmark. + +Note that this configuration is still incomplete for the amount of the variable TPCC varies. This intentional as we +want expose small amounts of independent variables until Dolt gets more robust. See `config.go` to get a breakdown of all the +variables TPCC varies. diff --git a/go/performance/utils/sysbench_runner/doltgres.go b/go/performance/utils/sysbench_runner/doltgres.go index 6713b1ca30e..9d962982e4a 100644 --- a/go/performance/utils/sysbench_runner/doltgres.go +++ b/go/performance/utils/sysbench_runner/doltgres.go @@ -125,7 +125,7 @@ func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Result results = make(Results, 0) for i := 0; i < b.config.Runs; i++ { for _, test := range tests { - tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) + tester := NewSysbenchTester(b.config, b.serverConfig, test, serverParams, stampFunc) var r *Result r, err = tester.Test(ctx) if err != nil { diff --git a/go/performance/utils/sysbench_runner/postgres.go b/go/performance/utils/sysbench_runner/postgres.go index 7024969d5f1..fa1416d334f 100644 --- a/go/performance/utils/sysbench_runner/postgres.go +++ b/go/performance/utils/sysbench_runner/postgres.go @@ -133,7 +133,7 @@ func (b *postgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Result results = make(Results, 0) for i := 0; i < b.config.Runs; i++ { for _, test := range tests { - tester := NewSysbenchTester(b.config, b.serverConfig, test, stampFunc) + tester := NewSysbenchTester(b.config, b.serverConfig, test, serverParams, stampFunc) var r *Result r, err = tester.Test(ctx) if err != nil { diff --git a/go/performance/utils/sysbench_runner/profile.go b/go/performance/utils/sysbench_runner/profile.go index f6b15e0200b..877ed41ed29 100644 --- a/go/performance/utils/sysbench_runner/profile.go +++ b/go/performance/utils/sysbench_runner/profile.go @@ -42,7 +42,7 @@ func (p *doltProfilerImpl) checkInstallation(ctx context.Context) error { } func (p *doltProfilerImpl) initDoltRepo(ctx context.Context) (string, error) { - return InitDoltRepo(ctx, p.dir, p.serverConfig.ServerExec, p.config.NomsBinFormat) + return InitDoltRepo(ctx, p.dir, p.serverConfig.ServerExec, p.config.NomsBinFormat, dbName) } func (p *doltProfilerImpl) Profile(ctx context.Context) error { @@ -92,7 +92,7 @@ func (p *doltProfilerImpl) Profile(ctx context.Context) error { results := make(Results, 0) for i := 0; i < p.config.Runs; i++ { for _, test := range tests { - tester := NewSysbenchTester(p.config, p.serverConfig, test, stampFunc) + tester := NewSysbenchTester(p.config, p.serverConfig, test, profileParams, stampFunc) r, err := tester.Test(ctx) if err != nil { server.Stop() diff --git a/go/performance/utils/sysbench_runner/tpcc_config.go b/go/performance/utils/sysbench_runner/tpcc_config.go index 1dced53895f..2e275a5631c 100644 --- a/go/performance/utils/sysbench_runner/tpcc_config.go +++ b/go/performance/utils/sysbench_runner/tpcc_config.go @@ -36,7 +36,7 @@ const ( ) var defaultTpccParams = []string{ - fmt.Sprintf("--mysql-db=%s", dbName), + fmt.Sprintf("--mysql-db=%s", tpccDbName), "--db-driver=mysql", } diff --git a/go/performance/utils/tpcc_runner/cmd/main.go b/go/performance/utils/tpcc_runner/cmd/main.go index a81be869fdf..9cfd04532f7 100644 --- a/go/performance/utils/tpcc_runner/cmd/main.go +++ b/go/performance/utils/tpcc_runner/cmd/main.go @@ -14,46 +14,46 @@ package main -import ( - "flag" - "fmt" - "log" - "os" - "path/filepath" - - tpcc_runner "github.com/dolthub/dolt/go/performance/utils/tpcc_runner" -) - -var configFile = flag.String("config", "", "path to config file q") - -func main() { - fmt.Println("Running the TPCC benchmark.") - - flag.Parse() - if *configFile == "" { - log.Fatal("Must supply config") - } - - configPath, err := filepath.Abs(*configFile) - if err != nil { - log.Fatal(err) - } - - if _, err = os.Stat(configPath); os.IsNotExist(err) { - log.Fatal(err) - } - - tpccBenchmarkConfig, err := tpcc_runner.FromFileConfig(configPath) - if err != nil { - log.Fatal(err) - } - - // Run the TPCC test - err = tpcc_runner.Run(tpccBenchmarkConfig) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - os.Exit(0) -} +//import ( +// "flag" +// "fmt" +// "log" +// "os" +// "path/filepath" +// +// tpcc_runner "github.com/dolthub/dolt/go/performance/utils/tpcc_runner" +//) +// +//var configFile = flag.String("config", "", "path to config file q") +// +//func main() { +// fmt.Println("Running the TPCC benchmark.") +// +// flag.Parse() +// if *configFile == "" { +// log.Fatal("Must supply config") +// } +// +// configPath, err := filepath.Abs(*configFile) +// if err != nil { +// log.Fatal(err) +// } +// +// if _, err = os.Stat(configPath); os.IsNotExist(err) { +// log.Fatal(err) +// } +// +// tpccBenchmarkConfig, err := tpcc_runner.FromFileConfig(configPath) +// if err != nil { +// log.Fatal(err) +// } +// +// // Run the TPCC test +// err = tpcc_runner.Run(tpccBenchmarkConfig) +// if err != nil { +// fmt.Println(err) +// os.Exit(1) +// } +// +// os.Exit(0) +//} diff --git a/go/performance/utils/tpcc_runner/config.go b/go/performance/utils/tpcc_runner/config.go index 7ffeeeba874..6fd863f14d2 100644 --- a/go/performance/utils/tpcc_runner/config.go +++ b/go/performance/utils/tpcc_runner/config.go @@ -1,255 +1,255 @@ -// Copyright 2022 Dolthub, Inc. +// // Copyright 2022 Dolthub, Inc. +// // +// // Licensed under the Apache License, Version 2.0 (the "License"); +// // you may not use this file except in compliance with the License. +// // You may obtain a copy of the License at +// // +// // http://www.apache.org/licenses/LICENSE-2.0 +// // +// // Unless required by applicable law or agreed to in writing, software +// // distributed under the License is distributed on an "AS IS" BASIS, +// // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// // See the License for the specific language governing permissions and +// // limitations under the License. +package tpcc_runner + // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//import ( +// "context" +// "encoding/json" +// "fmt" +// "os" +// "os/exec" +// "path/filepath" +// "runtime" // -// http://www.apache.org/licenses/LICENSE-2.0 +// "github.com/google/uuid" // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tpcc_runner - -import ( - "context" - "encoding/json" - "fmt" - "os" - "os/exec" - "path/filepath" - "runtime" - - "github.com/google/uuid" - - "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -const ( - defaultHost = "127.0.0.1" - defaultUser = "root" - - // Note this is built for the SysbenchDocker file. If you want to run locally you'll need to override these variables - // for your local MySQL setup. - tpccUserLocal = "'sysbench'@'localhost'" - tpccPassLocal = "sysbenchpass" -) - -var defaultTpccParams = []string{ - fmt.Sprintf("--mysql-db=%s", dbName), - "--db-driver=mysql", -} - -// TpccBenchmarkConfig represents a configuration for an execution of the TPCC Benchmark. It executes a series of tests -// against different ServerConfigurations. -type TpccBenchmarkConfig struct { - // RuntimeOS is the platform the benchmarks ran on - RuntimeOS string - - // RuntimeGoArch is the runtime architecture - RuntimeGoArch string - - // ScriptDir represents the location of the TPCC tests - ScriptDir string - - // Servers are the servers to benchmark. - Servers []*sysbench_runner.ServerConfig - - // ScaleFactors represent the scale at which to run each TpccBenchmark at. - ScaleFactors []int - - // NomsBinFormat specifies the NomsBinFormat - NomsBinFormat string -} - -func NewTpccConfig() *TpccBenchmarkConfig { - return &TpccBenchmarkConfig{ - Servers: make([]*sysbench_runner.ServerConfig, 0), - ScaleFactors: make([]int, 0), - } -} - -func (c *TpccBenchmarkConfig) updateDefaults() error { - if len(c.Servers) < 1 { - return sysbench_runner.ErrNoServersDefined - } - - // TODO: Eventually we need to support scale factors all the way to 10 - if len(c.ScaleFactors) == 0 { - c.ScaleFactors = append(c.ScaleFactors, 1) - } - - if c.RuntimeOS == "" { - c.RuntimeOS = runtime.GOOS - } - if c.RuntimeGoArch == "" { - c.RuntimeGoArch = runtime.GOARCH - } - - return c.validateServerConfigs() -} - -// validateServerConfigs ensures the ServerConfigs are valid -func (c *TpccBenchmarkConfig) validateServerConfigs() error { - portMap := make(map[int]sysbench_runner.ServerType) - for _, s := range c.Servers { - if s.Server != sysbench_runner.Dolt && s.Server != sysbench_runner.MySql { - return fmt.Errorf("unsupported server type: %s", s.Server) - } - - err := sysbench_runner.ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat) - if err != nil { - return err - } - - if s.Server == sysbench_runner.MySql { - err = sysbench_runner.CheckProtocol(s.ConnectionProtocol) - if err != nil { - return err - } - } - - if s.Host == "" { - s.Host = defaultHost - } - - portMap, err = sysbench_runner.CheckUpdatePortMap(s, portMap) - if err != nil { - return err - } - - err = sysbench_runner.CheckExec(s.ServerExec, "server exec") - if err != nil { - return err - } - } - return nil -} - -// FromFileConfig returns a validated Config based on the config file at the configPath -func FromFileConfig(configPath string) (*TpccBenchmarkConfig, error) { - data, err := os.ReadFile(configPath) - if err != nil { - return nil, err - } - - config := NewTpccConfig() - err = json.Unmarshal(data, config) - if err != nil { - return nil, err - } - - return config, nil -} - -// TpccTest encapsulates an End to End prepare, run, cleanup test case. -type TpccTest struct { - // Id represents a unique test id - Id string - - // Name represents the name of the test case - Name string - - // Params are associated parameters this test runs with - Params *TpccTestParams -} - -type TpccTestParams struct { - // NumThreads represents the number of threads running queries concurrently. - NumThreads int - - // ScaleFactor represents the number of warehouse to test this at scale. - ScaleFactor int - - // Tables represents the number of tables created per warehouse. - Tables int - - // TrxLevel represents what transaction level to use - TrxLevel string - - // ReportCSV determines whether to report output as a csv. - ReportCSV bool - - // ReportInterval defines how often the tpcc benchmark outputs performance stats. - ReportInterval int - - // Time represents how long - Time int -} - -// NewDefaultTpccParams returns default TpccTestParams. -func NewDefaultTpccParams() *TpccTestParams { - return &TpccTestParams{ - NumThreads: 2, // TODO: When ready, expose as command line argument. - ScaleFactor: 1, - Tables: 1, - TrxLevel: "RR", - ReportCSV: true, - ReportInterval: 1, - Time: 30, - } -} - -// NewTpccTest instantiates and returns a TPCC test. -func NewTpccTest(name string, params *TpccTestParams) *TpccTest { - return &TpccTest{ - Id: uuid.New().String(), - Name: name, - Params: params, - } -} - -// getArgs returns a test's args for all TPCC steps -func (t *TpccTest) getArgs(serverConfig *sysbench_runner.ServerConfig) []string { - params := make([]string, 0) - params = append(params, defaultTpccParams...) - - params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host)) - - // handle sysbench user for local mysql server - if serverConfig.Server == sysbench_runner.MySql && serverConfig.Host == defaultHost { - params = append(params, fmt.Sprintf("--mysql-user=%s", "sysbench")) - params = append(params, fmt.Sprintf("--mysql-password=%s", tpccPassLocal)) - } else { - params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port)) - params = append(params, fmt.Sprintf("--mysql-user=%s", defaultUser)) - } - - params = append(params, fmt.Sprintf("--time=%d", t.Params.Time)) - params = append(params, fmt.Sprintf("--threads=%d", t.Params.NumThreads)) - params = append(params, fmt.Sprintf("--report_interval=%d", t.Params.ReportInterval)) - params = append(params, fmt.Sprintf("--tables=%d", t.Params.Tables)) - params = append(params, fmt.Sprintf("--scale=%d", t.Params.ScaleFactor)) - params = append(params, fmt.Sprintf("--trx_level=%s", t.Params.TrxLevel)) - - return params -} - -// TpccPrepare prepares the command executable for the Prepare step. -func (t *TpccTest) TpccPrepare(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { - cmd := sysbench_runner.ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "prepare")...) - return addParamsToCmd(cmd, scriptDir) -} - -// TpccRun prepares the command executable for the Run step. -func (t *TpccTest) TpccRun(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { - cmd := exec.CommandContext(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "run")...) - return addParamsToCmd(cmd, scriptDir) -} - -// TpccCleanup prepares the cleanup executable for the Cleanup step. -func (t *TpccTest) TpccCleanup(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { - cmd := sysbench_runner.ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "cleanup")...) - return addParamsToCmd(cmd, scriptDir) -} - -func addParamsToCmd(cmd *exec.Cmd, scriptDir string) *exec.Cmd { - lp := filepath.Join(scriptDir, "?.lua") - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) - - return cmd -} +// "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" +//) +// +//const ( +// defaultHost = "127.0.0.1" +// defaultUser = "root" +// +// // Note this is built for the SysbenchDocker file. If you want to run locally you'll need to override these variables +// // for your local MySQL setup. +// tpccUserLocal = "'sysbench'@'localhost'" +// tpccPassLocal = "sysbenchpass" +//) +// +//var defaultTpccParams = []string{ +// fmt.Sprintf("--mysql-db=%s", dbName), +// "--db-driver=mysql", +//} +// +//// TpccBenchmarkConfig represents a configuration for an execution of the TPCC Benchmark. It executes a series of tests +//// against different ServerConfigurations. +//type TpccBenchmarkConfig struct { +// // RuntimeOS is the platform the benchmarks ran on +// RuntimeOS string +// +// // RuntimeGoArch is the runtime architecture +// RuntimeGoArch string +// +// // ScriptDir represents the location of the TPCC tests +// ScriptDir string +// +// // Servers are the servers to benchmark. +// Servers []*sysbench_runner.ServerConfig +// +// // ScaleFactors represent the scale at which to run each TpccBenchmark at. +// ScaleFactors []int +// +// // NomsBinFormat specifies the NomsBinFormat +// NomsBinFormat string +//} +// +//func NewTpccConfig() *TpccBenchmarkConfig { +// return &TpccBenchmarkConfig{ +// Servers: make([]*sysbench_runner.ServerConfig, 0), +// ScaleFactors: make([]int, 0), +// } +//} +// +//func (c *TpccBenchmarkConfig) updateDefaults() error { +// if len(c.Servers) < 1 { +// return sysbench_runner.ErrNoServersDefined +// } +// +// // TODO: Eventually we need to support scale factors all the way to 10 +// if len(c.ScaleFactors) == 0 { +// c.ScaleFactors = append(c.ScaleFactors, 1) +// } +// +// if c.RuntimeOS == "" { +// c.RuntimeOS = runtime.GOOS +// } +// if c.RuntimeGoArch == "" { +// c.RuntimeGoArch = runtime.GOARCH +// } +// +// return c.validateServerConfigs() +//} +// +//// validateServerConfigs ensures the ServerConfigs are valid +//func (c *TpccBenchmarkConfig) validateServerConfigs() error { +// portMap := make(map[int]sysbench_runner.ServerType) +// for _, s := range c.Servers { +// if s.Server != sysbench_runner.Dolt && s.Server != sysbench_runner.MySql { +// return fmt.Errorf("unsupported server type: %s", s.Server) +// } +// +// err := sysbench_runner.ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat) +// if err != nil { +// return err +// } +// +// if s.Server == sysbench_runner.MySql { +// err = sysbench_runner.CheckProtocol(s.ConnectionProtocol) +// if err != nil { +// return err +// } +// } +// +// if s.Host == "" { +// s.Host = defaultHost +// } +// +// portMap, err = sysbench_runner.CheckUpdatePortMap(s, portMap) +// if err != nil { +// return err +// } +// +// err = sysbench_runner.CheckExec(s.ServerExec, "server exec") +// if err != nil { +// return err +// } +// } +// return nil +//} +// +//// FromFileConfig returns a validated Config based on the config file at the configPath +//func FromFileConfig(configPath string) (*TpccBenchmarkConfig, error) { +// data, err := os.ReadFile(configPath) +// if err != nil { +// return nil, err +// } +// +// config := NewTpccConfig() +// err = json.Unmarshal(data, config) +// if err != nil { +// return nil, err +// } +// +// return config, nil +//} +// +//// TpccTest encapsulates an End to End prepare, run, cleanup test case. +//type TpccTest struct { +// // Id represents a unique test id +// Id string +// +// // Name represents the name of the test case +// Name string +// +// // Params are associated parameters this test runs with +// Params *TpccTestParams +//} +// +//type TpccTestParams struct { +// // NumThreads represents the number of threads running queries concurrently. +// NumThreads int +// +// // ScaleFactor represents the number of warehouse to test this at scale. +// ScaleFactor int +// +// // Tables represents the number of tables created per warehouse. +// Tables int +// +// // TrxLevel represents what transaction level to use +// TrxLevel string +// +// // ReportCSV determines whether to report output as a csv. +// ReportCSV bool +// +// // ReportInterval defines how often the tpcc benchmark outputs performance stats. +// ReportInterval int +// +// // Time represents how long +// Time int +//} +// +//// NewDefaultTpccParams returns default TpccTestParams. +//func NewDefaultTpccParams() *TpccTestParams { +// return &TpccTestParams{ +// NumThreads: 2, // TODO: When ready, expose as command line argument. +// ScaleFactor: 1, +// Tables: 1, +// TrxLevel: "RR", +// ReportCSV: true, +// ReportInterval: 1, +// Time: 30, +// } +//} +// +//// NewTpccTest instantiates and returns a TPCC test. +//func NewTpccTest(name string, params *TpccTestParams) *TpccTest { +// return &TpccTest{ +// Id: uuid.New().String(), +// Name: name, +// Params: params, +// } +//} +// +//// getArgs returns a test's args for all TPCC steps +//func (t *TpccTest) getArgs(serverConfig *sysbench_runner.ServerConfig) []string { +// params := make([]string, 0) +// params = append(params, defaultTpccParams...) +// +// params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host)) +// +// // handle sysbench user for local mysql server +// if serverConfig.Server == sysbench_runner.MySql && serverConfig.Host == defaultHost { +// params = append(params, fmt.Sprintf("--mysql-user=%s", "sysbench")) +// params = append(params, fmt.Sprintf("--mysql-password=%s", tpccPassLocal)) +// } else { +// params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port)) +// params = append(params, fmt.Sprintf("--mysql-user=%s", defaultUser)) +// } +// +// params = append(params, fmt.Sprintf("--time=%d", t.Params.Time)) +// params = append(params, fmt.Sprintf("--threads=%d", t.Params.NumThreads)) +// params = append(params, fmt.Sprintf("--report_interval=%d", t.Params.ReportInterval)) +// params = append(params, fmt.Sprintf("--tables=%d", t.Params.Tables)) +// params = append(params, fmt.Sprintf("--scale=%d", t.Params.ScaleFactor)) +// params = append(params, fmt.Sprintf("--trx_level=%s", t.Params.TrxLevel)) +// +// return params +//} +// +//// TpccPrepare prepares the command executable for the Prepare step. +//func (t *TpccTest) TpccPrepare(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { +// cmd := sysbench_runner.ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "prepare")...) +// return addParamsToCmd(cmd, scriptDir) +//} +// +//// TpccRun prepares the command executable for the Run step. +//func (t *TpccTest) TpccRun(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { +// cmd := exec.CommandContext(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "run")...) +// return addParamsToCmd(cmd, scriptDir) +//} +// +//// TpccCleanup prepares the cleanup executable for the Cleanup step. +//func (t *TpccTest) TpccCleanup(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { +// cmd := sysbench_runner.ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "cleanup")...) +// return addParamsToCmd(cmd, scriptDir) +//} +// +//func addParamsToCmd(cmd *exec.Cmd, scriptDir string) *exec.Cmd { +// lp := filepath.Join(scriptDir, "?.lua") +// cmd.Env = os.Environ() +// cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) +// +// return cmd +//} diff --git a/go/performance/utils/tpcc_runner/dolt.go b/go/performance/utils/tpcc_runner/dolt.go index e9ab273cf70..d76791bc557 100644 --- a/go/performance/utils/tpcc_runner/dolt.go +++ b/go/performance/utils/tpcc_runner/dolt.go @@ -14,191 +14,175 @@ package tpcc_runner -import ( - "context" - "fmt" - "os" - "os/exec" - "os/signal" - "path/filepath" - "sync" - "syscall" - "time" - - "golang.org/x/sync/errgroup" - - "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -const ( - dbName = "sbt" - nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" -) - -// BenchmarkDolt executes a set of tpcc tests against a dolt server. -func BenchmarkDolt(ctx context.Context, tppcConfig *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig) (sysbench_runner.Results, error) { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return nil, err - } - - err = sysbench_runner.UpdateDoltConfig(ctx, serverConfig.ServerExec) - if err != nil { - return nil, err - } - - testRepo, err := initDoltRepo(ctx, serverConfig, tppcConfig.NomsBinFormat) - if err != nil { - return nil, err - } - - withKeyCtx, cancel := context.WithCancel(ctx) - gServer, serverCtx := errgroup.WithContext(withKeyCtx) - - server := getDoltServer(serverCtx, serverConfig, testRepo, serverParams) - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - // launch the dolt server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(5 * time.Second) - - // GetTests and Benchmarks - tests := getTests(tppcConfig) - results := make(sysbench_runner.Results, 0) - - for _, test := range tests { - result, err := benchmark(ctx, test, serverConfig, tppcConfig) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - - results = append(results, result) - } - - // send signal to dolt server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - close(quit) - wg.Wait() - return nil, err - } - } - - close(quit) - wg.Wait() - - return results, os.RemoveAll(testRepo) -} - -// initDoltRepo initializes a dolt repo and returns the repo path -func initDoltRepo(ctx context.Context, config *sysbench_runner.ServerConfig, nbf string) (string, error) { - if nbf != "" { - if err := os.Setenv(nbfEnvVar, nbf); err != nil { - return "", err - } - } - - cwd, err := os.Getwd() - if err != nil { - return "", err - } - - testRepo := filepath.Join(cwd, dbName) - err = os.MkdirAll(testRepo, os.ModePerm) - if err != nil { - return "", err - } - - err = os.RemoveAll(filepath.Join(testRepo, ".dolt")) - if err != nil { - return "", err - } - - doltInit := sysbench_runner.ExecCommand(ctx, config.ServerExec, "init") - doltInit.Dir = testRepo - err = doltInit.Run() - if err != nil { - return "", err - } - - return testRepo, nil -} - -// getDoltServer returns a exec.Cmd for a dolt server -func getDoltServer(ctx context.Context, config *sysbench_runner.ServerConfig, testRepo string, params []string) *exec.Cmd { - server := sysbench_runner.ExecCommand(ctx, config.ServerExec, params...) - server.Dir = testRepo - return server -} - -// getTests creates a set of tests that the server needs to be executed on. -func getTests(config *TpccBenchmarkConfig) []*TpccTest { - tests := make([]*TpccTest, 0) - for _, sf := range config.ScaleFactors { - params := NewDefaultTpccParams() - params.ScaleFactor = sf - test := NewTpccTest(fmt.Sprintf("tpcc-scale-factor-%d", sf), params) - tests = append(tests, test) - } - - return tests -} - -// benchmark runs the relevant tpcc test against a server with a config. -func benchmark(ctx context.Context, test *TpccTest, serverConfig *sysbench_runner.ServerConfig, config *TpccBenchmarkConfig) (*sysbench_runner.Result, error) { - prepare := test.TpccPrepare(ctx, serverConfig, config.ScriptDir) - run := test.TpccRun(ctx, serverConfig, config.ScriptDir) - cleanup := test.TpccCleanup(ctx, serverConfig, config.ScriptDir) - - err := prepare.Run() - if err != nil { - return nil, err - } - - if run.Stdout != nil { - run.Stdout = nil - } - - out, err := run.Output() - if err != nil { - fmt.Print(string(out)) - return nil, err - } - - fmt.Print(string(out)) - - result, err := FromOutputResult(out, config, serverConfig, test, "tpcc", nil) - if err != nil { - return nil, err - } - - err = cleanup.Run() - if err != nil { - return nil, err - } - - return result, nil -} +//const ( +// dbName = "sbt" +// nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" +//) +// +//// BenchmarkDolt executes a set of tpcc tests against a dolt server. +//func BenchmarkDolt(ctx context.Context, tppcConfig *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig) (sysbench_runner.Results, error) { +// serverParams, err := serverConfig.GetServerArgs() +// if err != nil { +// return nil, err +// } +// +// err = sysbench_runner.UpdateDoltConfig(ctx, serverConfig.ServerExec) +// if err != nil { +// return nil, err +// } +// +// testRepo, err := initDoltRepo(ctx, serverConfig, tppcConfig.NomsBinFormat) +// if err != nil { +// return nil, err +// } +// +// withKeyCtx, cancel := context.WithCancel(ctx) +// gServer, serverCtx := errgroup.WithContext(withKeyCtx) +// +// server := getDoltServer(serverCtx, serverConfig, testRepo, serverParams) +// +// // handle user interrupt +// quit := make(chan os.Signal, 1) +// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) +// var wg sync.WaitGroup +// wg.Add(1) +// go func() { +// <-quit +// defer wg.Done() +// signal.Stop(quit) +// cancel() +// }() +// +// // launch the dolt server +// gServer.Go(func() error { +// return server.Run() +// }) +// +// // sleep to allow the server to start +// time.Sleep(5 * time.Second) +// +// // GetTests and Benchmarks +// tests := getTests(tppcConfig) +// results := make(sysbench_runner.Results, 0) +// +// for _, test := range tests { +// result, err := benchmark(ctx, test, serverConfig, tppcConfig) +// if err != nil { +// close(quit) +// wg.Wait() +// return nil, err +// } +// +// results = append(results, result) +// } +// +// // send signal to dolt server +// quit <- syscall.SIGTERM +// +// err = gServer.Wait() +// if err != nil { +// // we expect a kill error +// // we only exit in error if this is not the +// // error +// if err.Error() != "signal: killed" { +// close(quit) +// wg.Wait() +// return nil, err +// } +// } +// +// close(quit) +// wg.Wait() +// +// return results, os.RemoveAll(testRepo) +//} + +//// initDoltRepo initializes a dolt repo and returns the repo path +//func initDoltRepo(ctx context.Context, config *sysbench_runner.ServerConfig, nbf string) (string, error) { +// if nbf != "" { +// if err := os.Setenv(nbfEnvVar, nbf); err != nil { +// return "", err +// } +// } +// +// cwd, err := os.Getwd() +// if err != nil { +// return "", err +// } +// +// testRepo := filepath.Join(cwd, dbName) +// err = os.MkdirAll(testRepo, os.ModePerm) +// if err != nil { +// return "", err +// } +// +// err = os.RemoveAll(filepath.Join(testRepo, ".dolt")) +// if err != nil { +// return "", err +// } +// +// doltInit := sysbench_runner.ExecCommand(ctx, config.ServerExec, "init") +// doltInit.Dir = testRepo +// err = doltInit.Run() +// if err != nil { +// return "", err +// } +// +// return testRepo, nil +//} +// +//// getDoltServer returns a exec.Cmd for a dolt server +//func getDoltServer(ctx context.Context, config *sysbench_runner.ServerConfig, testRepo string, params []string) *exec.Cmd { +// server := sysbench_runner.ExecCommand(ctx, config.ServerExec, params...) +// server.Dir = testRepo +// return server +//} +// +//// getTests creates a set of tests that the server needs to be executed on. +//func getTests(config *TpccBenchmarkConfig) []*TpccTest { +// tests := make([]*TpccTest, 0) +// for _, sf := range config.ScaleFactors { +// params := NewDefaultTpccParams() +// params.ScaleFactor = sf +// test := NewTpccTest(fmt.Sprintf("tpcc-scale-factor-%d", sf), params) +// tests = append(tests, test) +// } +// +// return tests +//} +// +//// benchmark runs the relevant tpcc test against a server with a config. +//func benchmark(ctx context.Context, test *TpccTest, serverConfig *sysbench_runner.ServerConfig, config *TpccBenchmarkConfig) (*sysbench_runner.Result, error) { +// prepare := test.TpccPrepare(ctx, serverConfig, config.ScriptDir) +// run := test.TpccRun(ctx, serverConfig, config.ScriptDir) +// cleanup := test.TpccCleanup(ctx, serverConfig, config.ScriptDir) +// +// err := prepare.Run() +// if err != nil { +// return nil, err +// } +// +// if run.Stdout != nil { +// run.Stdout = nil +// } +// +// out, err := run.Output() +// if err != nil { +// fmt.Print(string(out)) +// return nil, err +// } +// +// fmt.Print(string(out)) +// +// result, err := FromOutputResult(out, config, serverConfig, test, "tpcc", nil) +// if err != nil { +// return nil, err +// } +// +// err = cleanup.Run() +// if err != nil { +// return nil, err +// } +// +// return result, nil +//} diff --git a/go/performance/utils/tpcc_runner/mysql.go b/go/performance/utils/tpcc_runner/mysql.go index ab4b33874b0..d80895dc961 100644 --- a/go/performance/utils/tpcc_runner/mysql.go +++ b/go/performance/utils/tpcc_runner/mysql.go @@ -14,122 +14,122 @@ package tpcc_runner -import ( - "context" - "fmt" - "os" - "os/exec" - "os/signal" - "sync" - "syscall" - "time" - - "golang.org/x/sync/errgroup" - - "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -// BenchmarkMysql benchmarks a mysql server based on the provided configurations. -func BenchmarkMysql(ctx context.Context, config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig) (sysbench_runner.Results, error) { - withKeyCtx, cancel := context.WithCancel(ctx) - - var serverDir string - defer func() { - if serverDir != "" { - os.RemoveAll(serverDir) - } - }() - - var localServer bool - var gServer *errgroup.Group - var serverCtx context.Context - var server *exec.Cmd - var err error - if serverConfig.Host == defaultHost { - localServer = true - serverDir, err = sysbench_runner.InitMysqlDataDir(ctx, serverConfig.ServerExec) - if err != nil { - cancel() - return nil, err - } - - gServer, serverCtx = errgroup.WithContext(withKeyCtx) - var serverParams []string - serverParams, err = serverConfig.GetServerArgs() - if err != nil { - cancel() - return nil, err - } - serverParams = append(serverParams, fmt.Sprintf("%s=%s", sysbench_runner.MysqlDataDirFlag, serverDir)) - server = getMysqlServer(serverCtx, serverConfig, serverParams) - - // launch the mysql server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(10 * time.Second) - - // setup mysqldb - err := sysbench_runner.SetupDB(ctx, sysbench_runner.GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName) - if err != nil { - cancel() - return nil, err - } - } - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - tests := getTests(config) - - results := make(sysbench_runner.Results, 0) - - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, serverConfig, config) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - results = append(results, r) - } - - // stop local mysql server - if localServer { - // send signal to server - quit <- syscall.SIGTERM - - err := gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - close(quit) - wg.Wait() - return nil, err - } - } - } - - close(quit) - wg.Wait() - - return results, nil -} - -// getMysqlServer returns a exec.Cmd for a dolt server -func getMysqlServer(ctx context.Context, config *sysbench_runner.ServerConfig, params []string) *exec.Cmd { - return sysbench_runner.ExecCommand(ctx, config.ServerExec, params...) -} +//import ( +// "context" +// "fmt" +// "os" +// "os/exec" +// "os/signal" +// "sync" +// "syscall" +// "time" +// +// "golang.org/x/sync/errgroup" +// +// "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" +//) +// +//// BenchmarkMysql benchmarks a mysql server based on the provided configurations. +//func BenchmarkMysql(ctx context.Context, config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig) (sysbench_runner.Results, error) { +// withKeyCtx, cancel := context.WithCancel(ctx) +// +// var serverDir string +// defer func() { +// if serverDir != "" { +// os.RemoveAll(serverDir) +// } +// }() +// +// var localServer bool +// var gServer *errgroup.Group +// var serverCtx context.Context +// var server *exec.Cmd +// var err error +// if serverConfig.Host == defaultHost { +// localServer = true +// serverDir, err = sysbench_runner.InitMysqlDataDir(ctx, serverConfig.ServerExec) +// if err != nil { +// cancel() +// return nil, err +// } +// +// gServer, serverCtx = errgroup.WithContext(withKeyCtx) +// var serverParams []string +// serverParams, err = serverConfig.GetServerArgs() +// if err != nil { +// cancel() +// return nil, err +// } +// serverParams = append(serverParams, fmt.Sprintf("%s=%s", sysbench_runner.MysqlDataDirFlag, serverDir)) +// server = getMysqlServer(serverCtx, serverConfig, serverParams) +// +// // launch the mysql server +// gServer.Go(func() error { +// return server.Run() +// }) +// +// // sleep to allow the server to start +// time.Sleep(10 * time.Second) +// +// // setup mysqldb +// err := sysbench_runner.SetupDB(ctx, sysbench_runner.GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName) +// if err != nil { +// cancel() +// return nil, err +// } +// } +// +// // handle user interrupt +// quit := make(chan os.Signal, 1) +// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) +// var wg sync.WaitGroup +// wg.Add(1) +// go func() { +// <-quit +// defer wg.Done() +// signal.Stop(quit) +// cancel() +// }() +// +// tests := getTests(config) +// +// results := make(sysbench_runner.Results, 0) +// +// for _, test := range tests { +// r, err := benchmark(withKeyCtx, test, serverConfig, config) +// if err != nil { +// close(quit) +// wg.Wait() +// return nil, err +// } +// results = append(results, r) +// } +// +// // stop local mysql server +// if localServer { +// // send signal to server +// quit <- syscall.SIGTERM +// +// err := gServer.Wait() +// if err != nil { +// // we expect a kill error +// // we only exit in error if this is not the +// // error +// if err.Error() != "signal: killed" { +// close(quit) +// wg.Wait() +// return nil, err +// } +// } +// } +// +// close(quit) +// wg.Wait() +// +// return results, nil +//} +// +//// getMysqlServer returns a exec.Cmd for a dolt server +//func getMysqlServer(ctx context.Context, config *sysbench_runner.ServerConfig, params []string) *exec.Cmd { +// return sysbench_runner.ExecCommand(ctx, config.ServerExec, params...) +//} diff --git a/go/performance/utils/tpcc_runner/results.go b/go/performance/utils/tpcc_runner/results.go index 33e0a11e262..6eaa8b866db 100644 --- a/go/performance/utils/tpcc_runner/results.go +++ b/go/performance/utils/tpcc_runner/results.go @@ -14,73 +14,73 @@ package tpcc_runner -import ( - "path/filepath" - "strings" - - "github.com/google/uuid" - - "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -// FromConfigsNewResult returns a new result with some fields set based on the provided configs -func FromConfigsNewResult(config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig, test *TpccTest, suiteId string, idFunc func() string) (*sysbench_runner.Result, error) { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return nil, err - } - - var getId func() string - if idFunc == nil { - getId = func() string { - return uuid.New().String() - } - } else { - getId = idFunc - } - - var name string - base := filepath.Base(test.Name) - ext := filepath.Ext(base) - name = strings.TrimSuffix(base, ext) - - return &sysbench_runner.Result{ - Id: getId(), - SuiteId: suiteId, - TestId: test.Id, - RuntimeOS: config.RuntimeOS, - RuntimeGoArch: config.RuntimeGoArch, - ServerName: string(serverConfig.Server), - ServerVersion: serverConfig.Version, - ServerParams: strings.Join(serverParams, " "), - TestName: name, - TestParams: strings.Join(test.getArgs(serverConfig), " "), - }, nil -} - -// FromOutputResult accepts raw sysbench run output and returns the Result -func FromOutputResult(output []byte, config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig, test *TpccTest, suiteId string, idFunc func() string) (*sysbench_runner.Result, error) { - result, err := FromConfigsNewResult(config, serverConfig, test, suiteId, idFunc) - if err != nil { - return nil, err - } - lines := strings.Split(string(output), "\n") - var process bool - for _, l := range lines { - trimmed := strings.TrimSpace(l) - if trimmed == "" { - continue - } - if strings.HasPrefix(trimmed, sysbench_runner.SqlStatsPrefix) { - process = true - continue - } - if process { - err := sysbench_runner.UpdateResult(result, trimmed) - if err != nil { - return result, err - } - } - } - return result, nil -} +//import ( +// "path/filepath" +// "strings" +// +// "github.com/google/uuid" +// +// "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" +//) +// +//// FromConfigsNewResult returns a new result with some fields set based on the provided configs +//func FromConfigsNewResult(config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig, test *TpccTest, suiteId string, idFunc func() string) (*sysbench_runner.Result, error) { +// serverParams, err := serverConfig.GetServerArgs() +// if err != nil { +// return nil, err +// } +// +// var getId func() string +// if idFunc == nil { +// getId = func() string { +// return uuid.New().String() +// } +// } else { +// getId = idFunc +// } +// +// var name string +// base := filepath.Base(test.Name) +// ext := filepath.Ext(base) +// name = strings.TrimSuffix(base, ext) +// +// return &sysbench_runner.Result{ +// Id: getId(), +// SuiteId: suiteId, +// TestId: test.Id, +// RuntimeOS: config.RuntimeOS, +// RuntimeGoArch: config.RuntimeGoArch, +// ServerName: string(serverConfig.Server), +// ServerVersion: serverConfig.Version, +// ServerParams: strings.Join(serverParams, " "), +// TestName: name, +// TestParams: strings.Join(test.getArgs(serverConfig), " "), +// }, nil +//} +// +//// FromOutputResult accepts raw sysbench run output and returns the Result +//func FromOutputResult(output []byte, config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig, test *TpccTest, suiteId string, idFunc func() string) (*sysbench_runner.Result, error) { +// result, err := FromConfigsNewResult(config, serverConfig, test, suiteId, idFunc) +// if err != nil { +// return nil, err +// } +// lines := strings.Split(string(output), "\n") +// var process bool +// for _, l := range lines { +// trimmed := strings.TrimSpace(l) +// if trimmed == "" { +// continue +// } +// if strings.HasPrefix(trimmed, sysbench_runner.SqlStatsPrefix) { +// process = true +// continue +// } +// if process { +// err := sysbench_runner.UpdateResult(result, trimmed) +// if err != nil { +// return result, err +// } +// } +// } +// return result, nil +//} diff --git a/go/performance/utils/tpcc_runner/run.go b/go/performance/utils/tpcc_runner/run.go index 59f78db3e89..72dfc7008f0 100644 --- a/go/performance/utils/tpcc_runner/run.go +++ b/go/performance/utils/tpcc_runner/run.go @@ -14,51 +14,44 @@ package tpcc_runner -import ( - "context" - "fmt" - - "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -func Run(config *TpccBenchmarkConfig) error { - err := config.updateDefaults() - if err != nil { - return err - } - - ctx := context.Background() - - for _, serverConfig := range config.Servers { - var results sysbench_runner.Results - var err error - switch serverConfig.Server { - case sysbench_runner.Dolt: - fmt.Println("Running Dolt Benchmark") - results, err = BenchmarkDolt(ctx, config, serverConfig) - if err != nil { - return err - } - case sysbench_runner.MySql: - fmt.Println("Running MySQL benchmark") - results, err = BenchmarkMysql(ctx, config, serverConfig) - if err != nil { - return err - } - default: - panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server)) - } - if err != nil { - return err - } - - err = sysbench_runner.WriteResults(serverConfig, results) - if err != nil { - return err - } - - fmt.Printf("Successfuly wrote results for %s\n", serverConfig.Server) - } - - return nil -} +//func Run(config *TpccBenchmarkConfig) error { +// err := config.updateDefaults() +// if err != nil { +// return err +// } +// +// ctx := context.Background() +// +// for _, serverConfig := range config.Servers { +// var results sysbench_runner.Results +// var err error +// switch serverConfig.Server { +// case sysbench_runner.Dolt: +// fmt.Println("Running Dolt Benchmark") +// results, err = BenchmarkDolt(ctx, config, serverConfig) +// if err != nil { +// return err +// } +// case sysbench_runner.MySql: +// fmt.Println("Running MySQL benchmark") +// results, err = BenchmarkMysql(ctx, config, serverConfig) +// if err != nil { +// return err +// } +// default: +// panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server)) +// } +// if err != nil { +// return err +// } +// +// err = sysbench_runner.WriteResults(serverConfig, results) +// if err != nil { +// return err +// } +// +// fmt.Printf("Successfuly wrote results for %s\n", serverConfig.Server) +// } +// +// return nil +//} diff --git a/go/performance/utils/tpcc_runner/sample-tpcc-config.json b/go/performance/utils/tpcc_runner/sample-tpcc-config.json index 6acbfa64a23..b0f189a69a9 100644 --- a/go/performance/utils/tpcc_runner/sample-tpcc-config.json +++ b/go/performance/utils/tpcc_runner/sample-tpcc-config.json @@ -6,17 +6,17 @@ "Server": "dolt", "Version": "HEAD", "ResultsFormat": "csv", - "ServerExec": "/Users/vinairachakonda/go/bin/dolt" + "ServerExec": "/Users/dustin/go/bin/dolt" }, { "Server": "mysql", "Version": "8.0.22", "ResultsFormat": "csv", - "ServerExec": "/usr/local/bin/mysqld", + "ServerExec": "/opt/homebrew/bin/mysqld", "ConnectionProtocol": "tcp" } ], - "ScriptDir":"/Users/vinairachakonda/go/src/dolthub/sysbench-tpcc", + "ScriptDir":"/Users/dustin/src/sysbench-tpcc", "ScaleFactors": [1], "NomsBinFormat": "__DOLT__" } From afcfdd56ed6b72981e0a3b4545d3998a7d7cf3b5 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 12 Feb 2024 15:36:13 -0800 Subject: [PATCH 11/25] /go/performance/utils/sysbench_runner: wip, remove tpcc code --- .../utils/sysbench_runner/run_test.go | 3 +- go/performance/utils/tpcc_runner/README.md | 33 --- go/performance/utils/tpcc_runner/cmd/main.go | 59 ---- go/performance/utils/tpcc_runner/config.go | 255 ------------------ go/performance/utils/tpcc_runner/dolt.go | 188 ------------- go/performance/utils/tpcc_runner/mysql.go | 135 ---------- go/performance/utils/tpcc_runner/results.go | 86 ------ go/performance/utils/tpcc_runner/run.go | 57 ---- .../utils/tpcc_runner/sample-tpcc-config.json | 22 -- 9 files changed, 2 insertions(+), 836 deletions(-) delete mode 100644 go/performance/utils/tpcc_runner/README.md delete mode 100644 go/performance/utils/tpcc_runner/cmd/main.go delete mode 100644 go/performance/utils/tpcc_runner/config.go delete mode 100644 go/performance/utils/tpcc_runner/dolt.go delete mode 100644 go/performance/utils/tpcc_runner/mysql.go delete mode 100644 go/performance/utils/tpcc_runner/results.go delete mode 100644 go/performance/utils/tpcc_runner/run.go delete mode 100644 go/performance/utils/tpcc_runner/sample-tpcc-config.json diff --git a/go/performance/utils/sysbench_runner/run_test.go b/go/performance/utils/sysbench_runner/run_test.go index dd85db622c4..645de9ffb00 100644 --- a/go/performance/utils/sysbench_runner/run_test.go +++ b/go/performance/utils/sysbench_runner/run_test.go @@ -15,6 +15,7 @@ package sysbench_runner import ( + "context" "log" "os" "testing" @@ -52,7 +53,7 @@ func TestRunner(t *testing.T) { InitBigRepo: true, } - err = Run(conf) + err = Run(context.Background(), conf) if err != nil { log.Fatal(err) } diff --git a/go/performance/utils/tpcc_runner/README.md b/go/performance/utils/tpcc_runner/README.md deleted file mode 100644 index cbbbf426395..00000000000 --- a/go/performance/utils/tpcc_runner/README.md +++ /dev/null @@ -1,33 +0,0 @@ -TPCC runner is a tool for running TPCC tests against sql servers. These tests run against the -Percona Labs repo [here](https://github.com/Percona-Lab/sysbench-tpcc). - -The tool requires a json config file to run. - -```bash -$ go run cmd/main.go --config=sample-tpcc-config.json -``` - -Note to this run this locally you need to have the TPCC repo cloned. The `ScriptDir` variable should then be linked -to the path of the cloned repo. - -Configuration: - -```json -{ - "Servers": "[...]", - "ScriptDir":"/Users/vinairachakonda/go/src/dolthub/sysbench-tpcc", - "ScaleFactors": [1] -} -``` - -`Servers`: The server defintions to run the benchmark against. Accepts Dolt and MySQL configuratiosn. - -`ScriptDir`: The directory of the TPCC testing scripts - -`ScaleFactors`: The number of warehouse to be generated in the test case. - -`NomsBinFormat`: The NomsBinFormat to use for this benchmark. - -Note that this configuration is still incomplete for the amount of the variable TPCC varies. This intentional as we -want expose small amounts of independent variables until Dolt gets more robust. See `config.go` to get a breakdown of all the -variables TPCC varies. diff --git a/go/performance/utils/tpcc_runner/cmd/main.go b/go/performance/utils/tpcc_runner/cmd/main.go deleted file mode 100644 index 9cfd04532f7..00000000000 --- a/go/performance/utils/tpcc_runner/cmd/main.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -//import ( -// "flag" -// "fmt" -// "log" -// "os" -// "path/filepath" -// -// tpcc_runner "github.com/dolthub/dolt/go/performance/utils/tpcc_runner" -//) -// -//var configFile = flag.String("config", "", "path to config file q") -// -//func main() { -// fmt.Println("Running the TPCC benchmark.") -// -// flag.Parse() -// if *configFile == "" { -// log.Fatal("Must supply config") -// } -// -// configPath, err := filepath.Abs(*configFile) -// if err != nil { -// log.Fatal(err) -// } -// -// if _, err = os.Stat(configPath); os.IsNotExist(err) { -// log.Fatal(err) -// } -// -// tpccBenchmarkConfig, err := tpcc_runner.FromFileConfig(configPath) -// if err != nil { -// log.Fatal(err) -// } -// -// // Run the TPCC test -// err = tpcc_runner.Run(tpccBenchmarkConfig) -// if err != nil { -// fmt.Println(err) -// os.Exit(1) -// } -// -// os.Exit(0) -//} diff --git a/go/performance/utils/tpcc_runner/config.go b/go/performance/utils/tpcc_runner/config.go deleted file mode 100644 index 6fd863f14d2..00000000000 --- a/go/performance/utils/tpcc_runner/config.go +++ /dev/null @@ -1,255 +0,0 @@ -// // Copyright 2022 Dolthub, Inc. -// // -// // Licensed under the Apache License, Version 2.0 (the "License"); -// // you may not use this file except in compliance with the License. -// // You may obtain a copy of the License at -// // -// // http://www.apache.org/licenses/LICENSE-2.0 -// // -// // Unless required by applicable law or agreed to in writing, software -// // distributed under the License is distributed on an "AS IS" BASIS, -// // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// // See the License for the specific language governing permissions and -// // limitations under the License. -package tpcc_runner - -// -//import ( -// "context" -// "encoding/json" -// "fmt" -// "os" -// "os/exec" -// "path/filepath" -// "runtime" -// -// "github.com/google/uuid" -// -// "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -//) -// -//const ( -// defaultHost = "127.0.0.1" -// defaultUser = "root" -// -// // Note this is built for the SysbenchDocker file. If you want to run locally you'll need to override these variables -// // for your local MySQL setup. -// tpccUserLocal = "'sysbench'@'localhost'" -// tpccPassLocal = "sysbenchpass" -//) -// -//var defaultTpccParams = []string{ -// fmt.Sprintf("--mysql-db=%s", dbName), -// "--db-driver=mysql", -//} -// -//// TpccBenchmarkConfig represents a configuration for an execution of the TPCC Benchmark. It executes a series of tests -//// against different ServerConfigurations. -//type TpccBenchmarkConfig struct { -// // RuntimeOS is the platform the benchmarks ran on -// RuntimeOS string -// -// // RuntimeGoArch is the runtime architecture -// RuntimeGoArch string -// -// // ScriptDir represents the location of the TPCC tests -// ScriptDir string -// -// // Servers are the servers to benchmark. -// Servers []*sysbench_runner.ServerConfig -// -// // ScaleFactors represent the scale at which to run each TpccBenchmark at. -// ScaleFactors []int -// -// // NomsBinFormat specifies the NomsBinFormat -// NomsBinFormat string -//} -// -//func NewTpccConfig() *TpccBenchmarkConfig { -// return &TpccBenchmarkConfig{ -// Servers: make([]*sysbench_runner.ServerConfig, 0), -// ScaleFactors: make([]int, 0), -// } -//} -// -//func (c *TpccBenchmarkConfig) updateDefaults() error { -// if len(c.Servers) < 1 { -// return sysbench_runner.ErrNoServersDefined -// } -// -// // TODO: Eventually we need to support scale factors all the way to 10 -// if len(c.ScaleFactors) == 0 { -// c.ScaleFactors = append(c.ScaleFactors, 1) -// } -// -// if c.RuntimeOS == "" { -// c.RuntimeOS = runtime.GOOS -// } -// if c.RuntimeGoArch == "" { -// c.RuntimeGoArch = runtime.GOARCH -// } -// -// return c.validateServerConfigs() -//} -// -//// validateServerConfigs ensures the ServerConfigs are valid -//func (c *TpccBenchmarkConfig) validateServerConfigs() error { -// portMap := make(map[int]sysbench_runner.ServerType) -// for _, s := range c.Servers { -// if s.Server != sysbench_runner.Dolt && s.Server != sysbench_runner.MySql { -// return fmt.Errorf("unsupported server type: %s", s.Server) -// } -// -// err := sysbench_runner.ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat) -// if err != nil { -// return err -// } -// -// if s.Server == sysbench_runner.MySql { -// err = sysbench_runner.CheckProtocol(s.ConnectionProtocol) -// if err != nil { -// return err -// } -// } -// -// if s.Host == "" { -// s.Host = defaultHost -// } -// -// portMap, err = sysbench_runner.CheckUpdatePortMap(s, portMap) -// if err != nil { -// return err -// } -// -// err = sysbench_runner.CheckExec(s.ServerExec, "server exec") -// if err != nil { -// return err -// } -// } -// return nil -//} -// -//// FromFileConfig returns a validated Config based on the config file at the configPath -//func FromFileConfig(configPath string) (*TpccBenchmarkConfig, error) { -// data, err := os.ReadFile(configPath) -// if err != nil { -// return nil, err -// } -// -// config := NewTpccConfig() -// err = json.Unmarshal(data, config) -// if err != nil { -// return nil, err -// } -// -// return config, nil -//} -// -//// TpccTest encapsulates an End to End prepare, run, cleanup test case. -//type TpccTest struct { -// // Id represents a unique test id -// Id string -// -// // Name represents the name of the test case -// Name string -// -// // Params are associated parameters this test runs with -// Params *TpccTestParams -//} -// -//type TpccTestParams struct { -// // NumThreads represents the number of threads running queries concurrently. -// NumThreads int -// -// // ScaleFactor represents the number of warehouse to test this at scale. -// ScaleFactor int -// -// // Tables represents the number of tables created per warehouse. -// Tables int -// -// // TrxLevel represents what transaction level to use -// TrxLevel string -// -// // ReportCSV determines whether to report output as a csv. -// ReportCSV bool -// -// // ReportInterval defines how often the tpcc benchmark outputs performance stats. -// ReportInterval int -// -// // Time represents how long -// Time int -//} -// -//// NewDefaultTpccParams returns default TpccTestParams. -//func NewDefaultTpccParams() *TpccTestParams { -// return &TpccTestParams{ -// NumThreads: 2, // TODO: When ready, expose as command line argument. -// ScaleFactor: 1, -// Tables: 1, -// TrxLevel: "RR", -// ReportCSV: true, -// ReportInterval: 1, -// Time: 30, -// } -//} -// -//// NewTpccTest instantiates and returns a TPCC test. -//func NewTpccTest(name string, params *TpccTestParams) *TpccTest { -// return &TpccTest{ -// Id: uuid.New().String(), -// Name: name, -// Params: params, -// } -//} -// -//// getArgs returns a test's args for all TPCC steps -//func (t *TpccTest) getArgs(serverConfig *sysbench_runner.ServerConfig) []string { -// params := make([]string, 0) -// params = append(params, defaultTpccParams...) -// -// params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host)) -// -// // handle sysbench user for local mysql server -// if serverConfig.Server == sysbench_runner.MySql && serverConfig.Host == defaultHost { -// params = append(params, fmt.Sprintf("--mysql-user=%s", "sysbench")) -// params = append(params, fmt.Sprintf("--mysql-password=%s", tpccPassLocal)) -// } else { -// params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port)) -// params = append(params, fmt.Sprintf("--mysql-user=%s", defaultUser)) -// } -// -// params = append(params, fmt.Sprintf("--time=%d", t.Params.Time)) -// params = append(params, fmt.Sprintf("--threads=%d", t.Params.NumThreads)) -// params = append(params, fmt.Sprintf("--report_interval=%d", t.Params.ReportInterval)) -// params = append(params, fmt.Sprintf("--tables=%d", t.Params.Tables)) -// params = append(params, fmt.Sprintf("--scale=%d", t.Params.ScaleFactor)) -// params = append(params, fmt.Sprintf("--trx_level=%s", t.Params.TrxLevel)) -// -// return params -//} -// -//// TpccPrepare prepares the command executable for the Prepare step. -//func (t *TpccTest) TpccPrepare(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { -// cmd := sysbench_runner.ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "prepare")...) -// return addParamsToCmd(cmd, scriptDir) -//} -// -//// TpccRun prepares the command executable for the Run step. -//func (t *TpccTest) TpccRun(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { -// cmd := exec.CommandContext(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "run")...) -// return addParamsToCmd(cmd, scriptDir) -//} -// -//// TpccCleanup prepares the cleanup executable for the Cleanup step. -//func (t *TpccTest) TpccCleanup(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { -// cmd := sysbench_runner.ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "cleanup")...) -// return addParamsToCmd(cmd, scriptDir) -//} -// -//func addParamsToCmd(cmd *exec.Cmd, scriptDir string) *exec.Cmd { -// lp := filepath.Join(scriptDir, "?.lua") -// cmd.Env = os.Environ() -// cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) -// -// return cmd -//} diff --git a/go/performance/utils/tpcc_runner/dolt.go b/go/performance/utils/tpcc_runner/dolt.go deleted file mode 100644 index d76791bc557..00000000000 --- a/go/performance/utils/tpcc_runner/dolt.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tpcc_runner - -//const ( -// dbName = "sbt" -// nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" -//) -// -//// BenchmarkDolt executes a set of tpcc tests against a dolt server. -//func BenchmarkDolt(ctx context.Context, tppcConfig *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig) (sysbench_runner.Results, error) { -// serverParams, err := serverConfig.GetServerArgs() -// if err != nil { -// return nil, err -// } -// -// err = sysbench_runner.UpdateDoltConfig(ctx, serverConfig.ServerExec) -// if err != nil { -// return nil, err -// } -// -// testRepo, err := initDoltRepo(ctx, serverConfig, tppcConfig.NomsBinFormat) -// if err != nil { -// return nil, err -// } -// -// withKeyCtx, cancel := context.WithCancel(ctx) -// gServer, serverCtx := errgroup.WithContext(withKeyCtx) -// -// server := getDoltServer(serverCtx, serverConfig, testRepo, serverParams) -// -// // handle user interrupt -// quit := make(chan os.Signal, 1) -// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) -// var wg sync.WaitGroup -// wg.Add(1) -// go func() { -// <-quit -// defer wg.Done() -// signal.Stop(quit) -// cancel() -// }() -// -// // launch the dolt server -// gServer.Go(func() error { -// return server.Run() -// }) -// -// // sleep to allow the server to start -// time.Sleep(5 * time.Second) -// -// // GetTests and Benchmarks -// tests := getTests(tppcConfig) -// results := make(sysbench_runner.Results, 0) -// -// for _, test := range tests { -// result, err := benchmark(ctx, test, serverConfig, tppcConfig) -// if err != nil { -// close(quit) -// wg.Wait() -// return nil, err -// } -// -// results = append(results, result) -// } -// -// // send signal to dolt server -// quit <- syscall.SIGTERM -// -// err = gServer.Wait() -// if err != nil { -// // we expect a kill error -// // we only exit in error if this is not the -// // error -// if err.Error() != "signal: killed" { -// close(quit) -// wg.Wait() -// return nil, err -// } -// } -// -// close(quit) -// wg.Wait() -// -// return results, os.RemoveAll(testRepo) -//} - -//// initDoltRepo initializes a dolt repo and returns the repo path -//func initDoltRepo(ctx context.Context, config *sysbench_runner.ServerConfig, nbf string) (string, error) { -// if nbf != "" { -// if err := os.Setenv(nbfEnvVar, nbf); err != nil { -// return "", err -// } -// } -// -// cwd, err := os.Getwd() -// if err != nil { -// return "", err -// } -// -// testRepo := filepath.Join(cwd, dbName) -// err = os.MkdirAll(testRepo, os.ModePerm) -// if err != nil { -// return "", err -// } -// -// err = os.RemoveAll(filepath.Join(testRepo, ".dolt")) -// if err != nil { -// return "", err -// } -// -// doltInit := sysbench_runner.ExecCommand(ctx, config.ServerExec, "init") -// doltInit.Dir = testRepo -// err = doltInit.Run() -// if err != nil { -// return "", err -// } -// -// return testRepo, nil -//} -// -//// getDoltServer returns a exec.Cmd for a dolt server -//func getDoltServer(ctx context.Context, config *sysbench_runner.ServerConfig, testRepo string, params []string) *exec.Cmd { -// server := sysbench_runner.ExecCommand(ctx, config.ServerExec, params...) -// server.Dir = testRepo -// return server -//} -// -//// getTests creates a set of tests that the server needs to be executed on. -//func getTests(config *TpccBenchmarkConfig) []*TpccTest { -// tests := make([]*TpccTest, 0) -// for _, sf := range config.ScaleFactors { -// params := NewDefaultTpccParams() -// params.ScaleFactor = sf -// test := NewTpccTest(fmt.Sprintf("tpcc-scale-factor-%d", sf), params) -// tests = append(tests, test) -// } -// -// return tests -//} -// -//// benchmark runs the relevant tpcc test against a server with a config. -//func benchmark(ctx context.Context, test *TpccTest, serverConfig *sysbench_runner.ServerConfig, config *TpccBenchmarkConfig) (*sysbench_runner.Result, error) { -// prepare := test.TpccPrepare(ctx, serverConfig, config.ScriptDir) -// run := test.TpccRun(ctx, serverConfig, config.ScriptDir) -// cleanup := test.TpccCleanup(ctx, serverConfig, config.ScriptDir) -// -// err := prepare.Run() -// if err != nil { -// return nil, err -// } -// -// if run.Stdout != nil { -// run.Stdout = nil -// } -// -// out, err := run.Output() -// if err != nil { -// fmt.Print(string(out)) -// return nil, err -// } -// -// fmt.Print(string(out)) -// -// result, err := FromOutputResult(out, config, serverConfig, test, "tpcc", nil) -// if err != nil { -// return nil, err -// } -// -// err = cleanup.Run() -// if err != nil { -// return nil, err -// } -// -// return result, nil -//} diff --git a/go/performance/utils/tpcc_runner/mysql.go b/go/performance/utils/tpcc_runner/mysql.go deleted file mode 100644 index d80895dc961..00000000000 --- a/go/performance/utils/tpcc_runner/mysql.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tpcc_runner - -//import ( -// "context" -// "fmt" -// "os" -// "os/exec" -// "os/signal" -// "sync" -// "syscall" -// "time" -// -// "golang.org/x/sync/errgroup" -// -// "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -//) -// -//// BenchmarkMysql benchmarks a mysql server based on the provided configurations. -//func BenchmarkMysql(ctx context.Context, config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig) (sysbench_runner.Results, error) { -// withKeyCtx, cancel := context.WithCancel(ctx) -// -// var serverDir string -// defer func() { -// if serverDir != "" { -// os.RemoveAll(serverDir) -// } -// }() -// -// var localServer bool -// var gServer *errgroup.Group -// var serverCtx context.Context -// var server *exec.Cmd -// var err error -// if serverConfig.Host == defaultHost { -// localServer = true -// serverDir, err = sysbench_runner.InitMysqlDataDir(ctx, serverConfig.ServerExec) -// if err != nil { -// cancel() -// return nil, err -// } -// -// gServer, serverCtx = errgroup.WithContext(withKeyCtx) -// var serverParams []string -// serverParams, err = serverConfig.GetServerArgs() -// if err != nil { -// cancel() -// return nil, err -// } -// serverParams = append(serverParams, fmt.Sprintf("%s=%s", sysbench_runner.MysqlDataDirFlag, serverDir)) -// server = getMysqlServer(serverCtx, serverConfig, serverParams) -// -// // launch the mysql server -// gServer.Go(func() error { -// return server.Run() -// }) -// -// // sleep to allow the server to start -// time.Sleep(10 * time.Second) -// -// // setup mysqldb -// err := sysbench_runner.SetupDB(ctx, sysbench_runner.GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName) -// if err != nil { -// cancel() -// return nil, err -// } -// } -// -// // handle user interrupt -// quit := make(chan os.Signal, 1) -// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) -// var wg sync.WaitGroup -// wg.Add(1) -// go func() { -// <-quit -// defer wg.Done() -// signal.Stop(quit) -// cancel() -// }() -// -// tests := getTests(config) -// -// results := make(sysbench_runner.Results, 0) -// -// for _, test := range tests { -// r, err := benchmark(withKeyCtx, test, serverConfig, config) -// if err != nil { -// close(quit) -// wg.Wait() -// return nil, err -// } -// results = append(results, r) -// } -// -// // stop local mysql server -// if localServer { -// // send signal to server -// quit <- syscall.SIGTERM -// -// err := gServer.Wait() -// if err != nil { -// // we expect a kill error -// // we only exit in error if this is not the -// // error -// if err.Error() != "signal: killed" { -// close(quit) -// wg.Wait() -// return nil, err -// } -// } -// } -// -// close(quit) -// wg.Wait() -// -// return results, nil -//} -// -//// getMysqlServer returns a exec.Cmd for a dolt server -//func getMysqlServer(ctx context.Context, config *sysbench_runner.ServerConfig, params []string) *exec.Cmd { -// return sysbench_runner.ExecCommand(ctx, config.ServerExec, params...) -//} diff --git a/go/performance/utils/tpcc_runner/results.go b/go/performance/utils/tpcc_runner/results.go deleted file mode 100644 index 6eaa8b866db..00000000000 --- a/go/performance/utils/tpcc_runner/results.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tpcc_runner - -//import ( -// "path/filepath" -// "strings" -// -// "github.com/google/uuid" -// -// "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -//) -// -//// FromConfigsNewResult returns a new result with some fields set based on the provided configs -//func FromConfigsNewResult(config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig, test *TpccTest, suiteId string, idFunc func() string) (*sysbench_runner.Result, error) { -// serverParams, err := serverConfig.GetServerArgs() -// if err != nil { -// return nil, err -// } -// -// var getId func() string -// if idFunc == nil { -// getId = func() string { -// return uuid.New().String() -// } -// } else { -// getId = idFunc -// } -// -// var name string -// base := filepath.Base(test.Name) -// ext := filepath.Ext(base) -// name = strings.TrimSuffix(base, ext) -// -// return &sysbench_runner.Result{ -// Id: getId(), -// SuiteId: suiteId, -// TestId: test.Id, -// RuntimeOS: config.RuntimeOS, -// RuntimeGoArch: config.RuntimeGoArch, -// ServerName: string(serverConfig.Server), -// ServerVersion: serverConfig.Version, -// ServerParams: strings.Join(serverParams, " "), -// TestName: name, -// TestParams: strings.Join(test.getArgs(serverConfig), " "), -// }, nil -//} -// -//// FromOutputResult accepts raw sysbench run output and returns the Result -//func FromOutputResult(output []byte, config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig, test *TpccTest, suiteId string, idFunc func() string) (*sysbench_runner.Result, error) { -// result, err := FromConfigsNewResult(config, serverConfig, test, suiteId, idFunc) -// if err != nil { -// return nil, err -// } -// lines := strings.Split(string(output), "\n") -// var process bool -// for _, l := range lines { -// trimmed := strings.TrimSpace(l) -// if trimmed == "" { -// continue -// } -// if strings.HasPrefix(trimmed, sysbench_runner.SqlStatsPrefix) { -// process = true -// continue -// } -// if process { -// err := sysbench_runner.UpdateResult(result, trimmed) -// if err != nil { -// return result, err -// } -// } -// } -// return result, nil -//} diff --git a/go/performance/utils/tpcc_runner/run.go b/go/performance/utils/tpcc_runner/run.go deleted file mode 100644 index 72dfc7008f0..00000000000 --- a/go/performance/utils/tpcc_runner/run.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tpcc_runner - -//func Run(config *TpccBenchmarkConfig) error { -// err := config.updateDefaults() -// if err != nil { -// return err -// } -// -// ctx := context.Background() -// -// for _, serverConfig := range config.Servers { -// var results sysbench_runner.Results -// var err error -// switch serverConfig.Server { -// case sysbench_runner.Dolt: -// fmt.Println("Running Dolt Benchmark") -// results, err = BenchmarkDolt(ctx, config, serverConfig) -// if err != nil { -// return err -// } -// case sysbench_runner.MySql: -// fmt.Println("Running MySQL benchmark") -// results, err = BenchmarkMysql(ctx, config, serverConfig) -// if err != nil { -// return err -// } -// default: -// panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server)) -// } -// if err != nil { -// return err -// } -// -// err = sysbench_runner.WriteResults(serverConfig, results) -// if err != nil { -// return err -// } -// -// fmt.Printf("Successfuly wrote results for %s\n", serverConfig.Server) -// } -// -// return nil -//} diff --git a/go/performance/utils/tpcc_runner/sample-tpcc-config.json b/go/performance/utils/tpcc_runner/sample-tpcc-config.json deleted file mode 100644 index b0f189a69a9..00000000000 --- a/go/performance/utils/tpcc_runner/sample-tpcc-config.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "Servers": [ - { - "Host": "127.0.0.1", - "Port": 3307, - "Server": "dolt", - "Version": "HEAD", - "ResultsFormat": "csv", - "ServerExec": "/Users/dustin/go/bin/dolt" - }, - { - "Server": "mysql", - "Version": "8.0.22", - "ResultsFormat": "csv", - "ServerExec": "/opt/homebrew/bin/mysqld", - "ConnectionProtocol": "tcp" - } - ], - "ScriptDir":"/Users/dustin/src/sysbench-tpcc", - "ScaleFactors": [1], - "NomsBinFormat": "__DOLT__" -} From ffecc7b7cfa0bbf941539e20e4733bbc2cff458a Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 12 Feb 2024 15:38:05 -0800 Subject: [PATCH 12/25] /go/performance/utils/sysbench_runner: wip, remove old benchmark code --- .../utils/sysbench_runner/dolt_old.go | 290 ------------------ .../utils/sysbench_runner/doltgres_old.go | 155 ---------- .../utils/sysbench_runner/mysql_old.go | 240 --------------- .../utils/sysbench_runner/postgres_old.go | 187 ----------- .../utils/sysbench_runner/profile_old.go | 169 ---------- 5 files changed, 1041 deletions(-) delete mode 100644 go/performance/utils/sysbench_runner/dolt_old.go delete mode 100644 go/performance/utils/sysbench_runner/doltgres_old.go delete mode 100644 go/performance/utils/sysbench_runner/mysql_old.go delete mode 100644 go/performance/utils/sysbench_runner/postgres_old.go delete mode 100644 go/performance/utils/sysbench_runner/profile_old.go diff --git a/go/performance/utils/sysbench_runner/dolt_old.go b/go/performance/utils/sysbench_runner/dolt_old.go deleted file mode 100644 index bf48627ca12..00000000000 --- a/go/performance/utils/sysbench_runner/dolt_old.go +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - - "github.com/dolthub/dolt/go/store/types" -) - -const ( -// dbName = "test" -// luaPath = "?.lua" -// bigEmptyRepo = "max-hoffman/big-empty" -// nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" -) - -//var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } - -//// BenchmarkDolt benchmarks dolt based on the provided configurations -//func BenchmarkDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { -// serverParams, err := serverConfig.GetServerArgs() -// if err != nil { -// return nil, err -// } -// -// err = DoltVersion(ctx, serverConfig.ServerExec) -// if err != nil { -// return nil, err -// } -// -// err = UpdateDoltConfig(ctx, serverConfig.ServerExec) -// if err != nil { -// return nil, err -// } -// -// testRepo, err := initDoltRepo(ctx, serverConfig, config.NomsBinFormat) -// if err != nil { -// return nil, err -// } -// -// withKeyCtx, cancel := context.WithCancel(ctx) -// gServer, serverCtx := errgroup.WithContext(withKeyCtx) -// -// server := getServer(serverCtx, serverConfig, testRepo, serverParams) -// -// // handle user interrupt -// quit := make(chan os.Signal, 1) -// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) -// var wg sync.WaitGroup -// wg.Add(1) -// go func() { -// <-quit -// defer wg.Done() -// signal.Stop(quit) -// cancel() -// }() -// -// // launch the dolt server -// gServer.Go(func() error { -// return server.Run() -// }) -// -// // sleep to allow the server to start -// time.Sleep(5 * time.Second) -// -// tests, err := GetTests(config, serverConfig, nil) -// if err != nil { -// return nil, err -// } -// -// results := make(Results, 0) -// for i := 0; i < config.Runs; i++ { -// for _, test := range tests { -// r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) -// if err != nil { -// close(quit) -// wg.Wait() -// return nil, err -// } -// results = append(results, r) -// } -// } -// -// // send signal to dolt server -// quit <- syscall.SIGTERM -// -// err = gServer.Wait() -// if err != nil { -// // we expect a kill error -// // we only exit in error if this is not the -// // error -// if err.Error() != "signal: killed" { -// fmt.Println(err) -// close(quit) -// wg.Wait() -// return nil, err -// } -// } -// -// fmt.Println("Successfully killed server") -// close(quit) -// wg.Wait() -// -// err = os.RemoveAll(testRepo) -// if err != nil { -// return nil, err -// } -// -// return results, nil -//} - -// DoltVersion ensures the dolt binary can run -func DoltVersion(ctx context.Context, serverExec string) error { - doltVersion := ExecCommand(ctx, serverExec, "version") - return doltVersion.Run() -} - -// initDoltRepo initializes a dolt repo and returns the repo path -func initDoltRepo(ctx context.Context, config *ServerConfig, nbf string) (string, error) { - cwd, err := os.Getwd() - if err != nil { - return "", err - } - - testRepo := filepath.Join(cwd, dbName) - if nbf == types.Format_LD_1.VersionString() { - err := ExecCommand(ctx, config.ServerExec, "clone", bigEmptyRepo, dbName).Run() - if err != nil { - return "", err - } - return testRepo, nil - } - err = os.MkdirAll(testRepo, os.ModePerm) - if err != nil { - return "", err - } - - if nbf != "" { - if err = os.Setenv(nbfEnvVar, nbf); err != nil { - return "", err - } - } - - doltInit := ExecCommand(ctx, config.ServerExec, "init") - doltInit.Dir = testRepo - err = doltInit.Run() - if err != nil { - return "", err - } - - return testRepo, nil -} - -// UpdateDoltConfig updates the dolt config if necessary -func UpdateDoltConfig(ctx context.Context, serverExec string) error { - err := checkSetDoltConfig(ctx, serverExec, "user.name", "benchmark") - if err != nil { - return err - } - return checkSetDoltConfig(ctx, serverExec, "user.email", "benchmark@dolthub.com") -} - -// checkSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary -func checkSetDoltConfig(ctx context.Context, serverExec, key, val string) error { - check := ExecCommand(ctx, serverExec, "config", "--global", "--get", key) - err := check.Run() - if err != nil { - // config get calls exit with 1 if not set - if err.Error() != "exit status 1" { - return err - } - - set := ExecCommand(ctx, serverExec, "config", "--global", "--add", key, val) - err := set.Run() - if err != nil { - return err - } - } - - return nil -} - -// getServer returns a exec.Cmd for a dolt server -func getServer(ctx context.Context, config *ServerConfig, testRepo string, params []string) *exec.Cmd { - server := ExecCommand(ctx, config.ServerExec, params...) - server.Dir = testRepo - return server -} - -// sysbenchPrepare returns a exec.Cmd for running the sysbench prepare step -func sysbenchPrepare(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { - cmd := exec.CommandContext(ctx, "sysbench", test.Prepare()...) - if test.FromScript { - lp := filepath.Join(scriptDir, luaPath) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) - } - return cmd -} - -// sysbenchRun returns a exec.Cmd for running the sysbench run step -func sysbenchRun(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { - cmd := exec.CommandContext(ctx, "sysbench", test.Run()...) - if test.FromScript { - lp := filepath.Join(scriptDir, luaPath) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) - } - return cmd -} - -// sysbenchPrepare returns a exec.Cmd for running the sysbench cleanup step -func sysbenchCleanup(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { - cmd := ExecCommand(ctx, "sysbench", test.Cleanup()...) - if test.FromScript { - lp := filepath.Join(scriptDir, luaPath) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) - } - return cmd -} - -// benchmark runs a sysbench benchmark against a server calling prepare, run, cleanup -func benchmark( - ctx context.Context, - test *Test, - config *Config, - serverConfig *ServerConfig, - stampFunc func() string, - suiteId string, -) (*Result, error) { - prepare := sysbenchPrepare(ctx, test, config.ScriptDir) - run := sysbenchRun(ctx, test, config.ScriptDir) - cleanup := sysbenchCleanup(ctx, test, config.ScriptDir) - - fmt.Println("Running test ", test.Name) - - out, err := prepare.Output() - if err != nil { - fmt.Println(string(out)) - return nil, err - } - - out, err = run.Output() - if err != nil { - fmt.Print(string(out)) - return nil, err - } - - if Debug == true { - fmt.Print(string(out)) - } - - r, err := FromOutputResult(out, config, serverConfig, test, suiteId, nil) - if err != nil { - return nil, err - } - - r.Stamp(stampFunc) - - return r, cleanup.Run() -} - -// fromChannelResults collects all Results from the given channel and returns them -func fromChannelResults(rc chan *Result) Results { - results := make(Results, 0) - for r := range rc { - if r != nil { - results = append(results, r) - } - } - return results -} diff --git a/go/performance/utils/sysbench_runner/doltgres_old.go b/go/performance/utils/sysbench_runner/doltgres_old.go deleted file mode 100644 index dfa8a1dc0ea..00000000000 --- a/go/performance/utils/sysbench_runner/doltgres_old.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -//import ( -// _ "github.com/lib/pq" -//) - -//// BenchmarkDoltgres benchmarks doltgres based on the provided configurations -//func BenchmarkDoltgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { -// serverParams, err := serverConfig.GetServerArgs() -// if err != nil { -// return nil, err -// } -// -// err = DoltVersion(ctx, serverConfig.ServerExec) -// if err != nil { -// return nil, err -// } -// -// serverDir, err := CreateServerDir(dbName) -// if err != nil { -// return nil, err -// } -// defer func() { -// cleanupDoltgresServerDir(serverDir) -// }() -// -// serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir)) -// -// withKeyCtx, cancel := context.WithCancel(ctx) -// gServer, serverCtx := errgroup.WithContext(withKeyCtx) -// -// server := getServer(serverCtx, serverConfig, serverDir, serverParams) -// -// // handle user interrupt -// quit := make(chan os.Signal, 1) -// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) -// var wg sync.WaitGroup -// wg.Add(1) -// go func() { -// <-quit -// defer wg.Done() -// signal.Stop(quit) -// cancel() -// }() -// -// // launch the dolt server -// gServer.Go(func() error { -// return server.Run() -// }) -// -// // sleep to allow the server to start -// time.Sleep(5 * time.Second) -// -// // create the db against the running server -// err = createDb(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "doltgres", dbName) -// if err != nil { -// close(quit) -// wg.Wait() -// return nil, err -// } -// -// tests, err := GetTests(config, serverConfig, nil) -// if err != nil { -// close(quit) -// wg.Wait() -// return nil, err -// } -// -// results := make(Results, 0) -// for i := 0; i < config.Runs; i++ { -// for _, test := range tests { -// r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) -// if err != nil { -// close(quit) -// wg.Wait() -// return nil, err -// } -// results = append(results, r) -// } -// } -// -// // send signal to dolt server -// quit <- syscall.SIGTERM -// -// err = gServer.Wait() -// if err != nil { -// // we expect a kill error -// // we only exit in error if this is not the -// // error -// if err.Error() != "signal: killed" { -// fmt.Println(err) -// close(quit) -// wg.Wait() -// return nil, err -// } -// } -// -// fmt.Println("Successfully killed server") -// close(quit) -// wg.Wait() -// -// return results, nil -//} - -//func createDb(ctx context.Context, host, port, user, dbname string) error { -// psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname) -// -// // open database -// db, err := sql.Open("postgres", psqlconn) -// if err != nil { -// return err -// } -// -// // close database -// defer db.Close() -// -// // check db -// err = db.PingContext(ctx) -// if err != nil { -// return err -// } -// -// _, err = db.ExecContext(ctx, fmt.Sprintf("create database %s;", dbname)) -// return err -//} - -//// cleanupDoltgresServerDir cleans up the doltgres assets in the provided dir -//func cleanupDoltgresServerDir(dir string) error { -// dataDir := filepath.Join(dir, ".dolt") -// defaultDir := filepath.Join(dir, "doltgres") -// testDir := filepath.Join(dir, dbName) -// for _, d := range []string{dataDir, defaultDir, testDir} { -// if _, err := os.Stat(d); !os.IsNotExist(err) { -// err = os.RemoveAll(d) -// if err != nil { -// return err -// } -// } -// } -// return nil -//} diff --git a/go/performance/utils/sysbench_runner/mysql_old.go b/go/performance/utils/sysbench_runner/mysql_old.go deleted file mode 100644 index 4d456afa2a0..00000000000 --- a/go/performance/utils/sysbench_runner/mysql_old.go +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -import ( - "context" - "database/sql" - "fmt" - - _ "github.com/go-sql-driver/mysql" -) - -type MysqlConfig struct { - Socket string - ConnectionProtocol string - Port int - Host string -} - -//// BenchmarkMysql benchmarks mysql based on the provided configurations -//func BenchmarkMysql(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { -// withKeyCtx, cancel := context.WithCancel(ctx) -// -// var serverDir string -// defer func() { -// if serverDir != "" { -// os.RemoveAll(serverDir) -// } -// }() -// -// var localServer bool -// var gServer *errgroup.Group -// var serverCtx context.Context -// var server *exec.Cmd -// var err error -// if serverConfig.Host == defaultHost { -// log.Println("Launching the default server") -// localServer = true -// -// serverDir, err = InitMysqlDataDir(ctx, serverConfig) -// if err != nil { -// cancel() -// return nil, err -// } -// -// gServer, serverCtx = errgroup.WithContext(withKeyCtx) -// var serverParams []string -// serverParams, err = serverConfig.GetServerArgs() -// if err != nil { -// cancel() -// return nil, err -// } -// serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) -// -// server = getMysqlServer(serverCtx, serverConfig, serverParams) -// -// // launch the mysql server -// gServer.Go(func() error { -// return server.Run() -// }) -// -// // sleep to allow the server to start -// time.Sleep(10 * time.Second) -// -// // setup mysqldb -// err := SetupDB(ctx, GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName) -// if err != nil { -// cancel() -// return nil, err -// } -// log.Println("Successfully set up the MySQL database") -// } -// -// // handle user interrupt -// quit := make(chan os.Signal, 1) -// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) -// var wg sync.WaitGroup -// wg.Add(1) -// go func() { -// <-quit -// defer wg.Done() -// signal.Stop(quit) -// cancel() -// }() -// -// tests, err := GetTests(config, serverConfig, nil) -// if err != nil { -// return nil, err -// } -// -// results := make(Results, 0) -// for i := 0; i < config.Runs; i++ { -// for _, test := range tests { -// r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) -// if err != nil { -// close(quit) -// wg.Wait() -// return nil, err -// } -// results = append(results, r) -// } -// } -// -// // stop local mysql server -// if localServer { -// // send signal to server -// quit <- syscall.SIGTERM -// -// err = gServer.Wait() -// if err != nil { -// // we expect a kill error -// // we only exit in error if this is not the -// // error -// if err.Error() != "signal: killed" { -// close(quit) -// wg.Wait() -// return nil, err -// } -// } -// } -// -// fmt.Println("Successfully killed server") -// close(quit) -// wg.Wait() -// -// return results, nil -//} - -//// getMysqlServer returns a exec.Cmd for a dolt server -//func getMysqlServer(ctx context.Context, config *ServerConfig, params []string) *exec.Cmd { -// return ExecCommand(ctx, config.ServerExec, params...) -//} - -//// InitMysqlDataDir initializes a mysql data dir and returns the path -//func InitMysqlDataDir(ctx context.Context, config *ServerConfig) (string, error) { -// serverDir, err := CreateServerDir(dbName) -// if err != nil { -// return "", err -// } -// -// msInit := ExecCommand(ctx, config.ServerExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) -// err = msInit.Run() -// if err != nil { -// return "", err -// } -// -// return serverDir, nil -//} - -func SetupDB(ctx context.Context, mConfig MysqlConfig, databaseName string) (err error) { - dsn, err := FormatDsn(mConfig) - if err != nil { - return err - } - - // TODO make sure this can work on windows - db, err := sql.Open("mysql", dsn) - if err != nil { - return err - } - defer func() { - rerr := db.Close() - if err == nil { - err = rerr - } - }() - err = db.Ping() - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", databaseName)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE %s", databaseName)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s", sysbenchUserLocal)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s'", sysbenchUserLocal, sysbenchPassLocal)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("GRANT ALL ON %s.* to %s", databaseName, sysbenchUserLocal)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, "SET GLOBAL local_infile = 'ON'") - if err != nil { - return err - } - // Required for running groupby_scan.lua without error - _, err = db.ExecContext(ctx, "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));") - if err != nil { - return err - } - - return -} - -func FormatDsn(mConfig MysqlConfig) (string, error) { - var socketPath string - if mConfig.Socket != "" { - socketPath = mConfig.Socket - } else { - socketPath = defaultMysqlSocket - } - - if mConfig.ConnectionProtocol == tcpProtocol { - return fmt.Sprintf("root@tcp(%s:%d)/", mConfig.Host, mConfig.Port), nil - } else if mConfig.ConnectionProtocol == unixProtocol { - return fmt.Sprintf("root@unix(%s)/", socketPath), nil - } else { - return "", ErrUnsupportedConnectionProtocol - } -} - -func GetMysqlConnectionConfigFromServerConfig(config *ServerConfig) MysqlConfig { - return MysqlConfig{ - Socket: config.Socket, - ConnectionProtocol: config.ConnectionProtocol, - Port: config.Port, - Host: defaultHost, - } -} diff --git a/go/performance/utils/sysbench_runner/postgres_old.go b/go/performance/utils/sysbench_runner/postgres_old.go deleted file mode 100644 index 10fcec03b29..00000000000 --- a/go/performance/utils/sysbench_runner/postgres_old.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -//import ( -// _ "github.com/lib/pq" -//) - -//type PostgresConfig struct { -// Socket string -// ConnectionProtocol string -// Port int -// Host string -//} - -//// BenchmarkPostgres benchmarks postgres based on the provided configurations -//func BenchmarkPostgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { -// withKeyCtx, cancel := context.WithCancel(ctx) -// -// var serverDir string -// defer func() { -// if serverDir != "" { -// os.RemoveAll(serverDir) -// } -// }() -// -// var localServer bool -// var gServer *errgroup.Group -// var serverCtx context.Context -// var server *exec.Cmd -// var err error -// if serverConfig.Host == defaultHost { -// log.Println("Launching the default server") -// localServer = true -// -// serverDir, err = initPostgresDataDir(ctx, serverConfig) -// if err != nil { -// cancel() -// return nil, err -// } -// gServer, serverCtx = errgroup.WithContext(withKeyCtx) -// var serverParams []string -// serverParams, err = serverConfig.GetServerArgs() -// if err != nil { -// cancel() -// return nil, err -// } -// serverParams = append(serverParams, "-D", serverDir) -// server = getMysqlServer(serverCtx, serverConfig, serverParams) -// server.Env = append(server.Env, "LC_ALL=C") -// -// // launch the postgres server -// gServer.Go(func() error { -// return server.Run() -// }) -// -// // sleep to allow the server to start -// time.Sleep(10 * time.Second) -// -// // setup postgres db -// err := setupPostgresDB(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "postgres", dbName) -// if err != nil { -// cancel() -// return nil, err -// } -// -// log.Println("Successfully set up the Postgres database") -// } -// -// // handle user interrupt -// quit := make(chan os.Signal, 1) -// signal.Notify(quit, os.Interrupt, syscall.SIGTERM) -// var wg sync.WaitGroup -// wg.Add(1) -// go func() { -// <-quit -// defer wg.Done() -// signal.Stop(quit) -// cancel() -// }() -// -// tests, err := GetTests(config, serverConfig, nil) -// if err != nil { -// return nil, err -// } -// -// results := make(Results, 0) -// for i := 0; i < config.Runs; i++ { -// for _, test := range tests { -// r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) -// if err != nil { -// close(quit) -// wg.Wait() -// return nil, err -// } -// results = append(results, r) -// } -// } -// -// // stop local mysql server -// if localServer { -// // send signal to server -// quit <- syscall.SIGTERM -// -// err = gServer.Wait() -// if err != nil { -// // we expect a kill error -// // we only exit in error if this is not the -// // error -// if err.Error() != "signal: killed" { -// close(quit) -// wg.Wait() -// return nil, err -// } -// } -// } -// -// fmt.Println("Successfully killed server") -// close(quit) -// wg.Wait() -// -// return results, nil -//} - -//// initPostgresDataDir initializes a postgres data dir and returns the path -//func initPostgresDataDir(ctx context.Context, config *ServerConfig) (string, error) { -// serverDir, err := CreateServerDir(dbName) -// if err != nil { -// return "", err -// } -// -// pgInit := ExecCommand(ctx, config.InitExec, fmt.Sprintf("--pgdata=%s", serverDir), "--username=postgres") -// err = pgInit.Run() -// if err != nil { -// return "", err -// } -// -// return serverDir, nil -//} - -//func setupPostgresDB(ctx context.Context, host, port, user, dbname string) (err error) { -// psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname) -// -// db, err := sql.Open("postgres", psqlconn) -// if err != nil { -// return err -// } -// defer func() { -// rerr := db.Close() -// if err == nil { -// err = rerr -// } -// }() -// err = db.PingContext(ctx) -// if err != nil { -// return err -// } -// _, err = db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", dbname)) -// if err != nil { -// return err -// } -// _, err = db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s", sysbenchUsername)) -// if err != nil { -// return err -// } -// _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE USER %s WITH PASSWORD '%s'", sysbenchUsername, sysbenchPassLocal)) -// if err != nil { -// return err -// } -// _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE %s WITH OWNER %s", dbname, sysbenchUsername)) -// if err != nil { -// return err -// } -// return -//} diff --git a/go/performance/utils/sysbench_runner/profile_old.go b/go/performance/utils/sysbench_runner/profile_old.go deleted file mode 100644 index 00867711ec4..00000000000 --- a/go/performance/utils/sysbench_runner/profile_old.go +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -//// ProfileDolt profiles dolt while running the provided tests -//func ProfileDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) error { -// serverParams, err := serverConfig.GetServerArgs() -// if err != nil { -// return err -// } -// -// err = DoltVersion(ctx, serverConfig.ServerExec) -// if err != nil { -// return err -// } -// -// err = UpdateDoltConfig(ctx, serverConfig.ServerExec) -// if err != nil { -// return err -// } -// -// testRepo, err := initDoltRepo(ctx, serverConfig, config.NomsBinFormat) -// if err != nil { -// return err -// } -// -// tests, err := GetTests(config, serverConfig, nil) -// if err != nil { -// return err -// } -// -// tempProfilesDir, err := os.MkdirTemp("", "") -// if err != nil { -// return err -// } -// defer os.RemoveAll(tempProfilesDir) -// -// for i := 0; i < config.Runs; i++ { -// for _, test := range tests { -// _, err = profileTest(ctx, test, config, serverConfig, serverParams, testRepo, tempProfilesDir) -// if err != nil { -// return err -// } -// } -// } -// -// profile, err := mergeProfiles(ctx, tempProfilesDir, serverConfig.ProfilePath) -// if err != nil { -// return err -// } -// -// fmt.Println("Profile created at:", profile) -// -// err = os.RemoveAll(testRepo) -// if err != nil { -// return err -// } -// -// return nil -//} - -//func profileTest(ctx context.Context, test *Test, config *Config, serverConfig *ServerConfig, serverParams []string, testRepo, profileDir string) (string, error) { -// profilePath, err := os.MkdirTemp("", filepath.Base(test.Name)) -// if err != nil { -// return "", err -// } -// defer os.RemoveAll(profilePath) -// -// tempProfile := filepath.Join(profilePath, cpuProfileFilename) -// profileParams := make([]string, 0) -// profileParams = append(profileParams, profileFlag, cpuProfile, profilePathFlag, profilePath) -// profileParams = append(profileParams, serverParams...) -// -// withKeyCtx, cancel := context.WithCancel(ctx) -// defer cancel() -// gServer, serverCtx := errgroup.WithContext(withKeyCtx) -// server := getServer(serverCtx, serverConfig, testRepo, profileParams) -// -// // handle user interrupt -// quit := make(chan os.Signal, 1) -// signal.Notify(quit, os.Interrupt, syscall.SIGINT) -// var wg sync.WaitGroup -// wg.Add(1) -// go func() { -// s := <-quit -// defer wg.Done() -// server.Process.Signal(s) -// signal.Stop(quit) -// }() -// -// // launch the dolt server -// gServer.Go(func() error { -// return server.Run() -// }) -// -// // sleep to allow the server to start -// time.Sleep(5 * time.Second) -// -// _, err = benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) -// if err != nil { -// close(quit) -// wg.Wait() -// return "", err -// } -// -// // send signal to dolt server -// quit <- syscall.SIGINT -// -// err = gServer.Wait() -// if err != nil { -// // we expect a kill error -// // we only exit in error if this is not the -// // error -// if err.Error() != "signal: killed" { -// fmt.Println(err) -// close(quit) -// wg.Wait() -// return "", err -// } -// } -// -// fmt.Println("Successfully killed server") -// close(quit) -// wg.Wait() -// -// info, err := os.Stat(tempProfile) -// if err != nil { -// return "", err -// } -// if info.Size() < 1 { -// return "", fmt.Errorf("failed to create profile: file was empty") -// } -// -// finalProfile := filepath.Join(profileDir, fmt.Sprintf("%s_%s_%s", serverConfig.Id, test.Name, cpuProfileFilename)) -// err = moveFile(tempProfile, finalProfile) -// return finalProfile, err -//} - -//func mergeProfiles(ctx context.Context, sourceProfilesDir, destProfileDir string) (string, error) { -// tmp, err := os.MkdirTemp("", "final_cpu_pprof") -// if err != nil { -// return "", err -// } -// defer os.RemoveAll(tmp) -// outfile := filepath.Join(tmp, "cpu.pprof") -// -// merge := ExecCommand(ctx, "/bin/sh", "-c", fmt.Sprintf("go tool pprof -proto *.pprof > %s", outfile)) -// merge.Dir = sourceProfilesDir -// err = merge.Run() -// if err != nil { -// return "", err -// } -// -// final := filepath.Join(destProfileDir, "cpu.pprof") -// err = moveFile(outfile, final) -// return final, err -//} From 25c78c33c0939da751a3b1aeb1cb32303fff6e7f Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 12 Feb 2024 17:23:35 -0800 Subject: [PATCH 13/25] /go/performance/utils/sysbench_runner: wip, working through config refactor --- .../utils/sysbench_runner/config.go | 616 +----------------- go/performance/utils/sysbench_runner/dolt.go | 6 +- .../sysbench_runner/dolt_server_config.go | 141 ++++ .../utils/sysbench_runner/dolt_tpcc.go | 4 +- .../utils/sysbench_runner/doltgres.go | 8 +- .../sysbench_runner/doltgres_server_config.go | 102 +++ go/performance/utils/sysbench_runner/mysql.go | 6 +- .../sysbench_runner/mysql_server_config.go | 122 ++++ .../utils/sysbench_runner/mysql_tpcc.go | 4 +- .../utils/sysbench_runner/postgres.go | 8 +- .../sysbench_runner/postgres_server_config.go | 111 ++++ .../utils/sysbench_runner/profile.go | 6 +- .../utils/sysbench_runner/results.go | 66 -- .../utils/sysbench_runner/results_test.go | 18 +- go/performance/utils/sysbench_runner/run.go | 4 +- .../utils/sysbench_runner/run_test.go | 10 +- .../utils/sysbench_runner/server.go | 4 +- .../utils/sysbench_runner/server_config.go | 23 + .../utils/sysbench_runner/sysbench.go | 26 +- .../utils/sysbench_runner/sysbench_config.go | 510 +++++++++++++++ ...config_test.go => sysbench_config_test.go} | 48 +- .../utils/sysbench_runner/sysbench_tests.go | 47 ++ .../utils/sysbench_runner/test_config.go | 69 ++ .../utils/sysbench_runner/tester.go | 6 + go/performance/utils/sysbench_runner/tpcc.go | 4 +- .../utils/sysbench_runner/tpcc_config.go | 14 +- 26 files changed, 1220 insertions(+), 763 deletions(-) create mode 100644 go/performance/utils/sysbench_runner/dolt_server_config.go create mode 100644 go/performance/utils/sysbench_runner/doltgres_server_config.go create mode 100644 go/performance/utils/sysbench_runner/mysql_server_config.go create mode 100644 go/performance/utils/sysbench_runner/postgres_server_config.go create mode 100644 go/performance/utils/sysbench_runner/server_config.go create mode 100644 go/performance/utils/sysbench_runner/sysbench_config.go rename go/performance/utils/sysbench_runner/{config_test.go => sysbench_config_test.go} (69%) create mode 100644 go/performance/utils/sysbench_runner/sysbench_tests.go create mode 100644 go/performance/utils/sysbench_runner/test_config.go diff --git a/go/performance/utils/sysbench_runner/config.go b/go/performance/utils/sysbench_runner/config.go index 39a69aceac4..42d21ec2fe9 100644 --- a/go/performance/utils/sysbench_runner/config.go +++ b/go/performance/utils/sysbench_runner/config.go @@ -1,616 +1,8 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package sysbench_runner -import ( - "encoding/json" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "runtime" - - "github.com/google/uuid" -) - -const ( - Dolt ServerType = "dolt" - Doltgres ServerType = "doltgres" - Postgres ServerType = "postgres" - MySql ServerType = "mysql" - - CsvFormat = "csv" - JsonFormat = "json" - - CsvExt = ".csv" - JsonExt = ".json" - - defaultHost = "127.0.0.1" - defaultPort = 3306 - - defaultMysqlSocket = "/var/run/mysqld/mysqld.sock" - - tcpProtocol = "tcp" - unixProtocol = "unix" - - sysbenchUsername = "sysbench" - sysbenchUserLocal = "'sysbench'@'localhost'" - sysbenchPassLocal = "sysbenchpass" - - userFlag = "--user" - hostFlag = "--host" - portFlag = "--port" - skipBinLogFlag = "--skip-log-bin" - profileFlag = "--prof" - profilePathFlag = "--prof-path" - cpuProfile = "cpu" - doltgresDataDirFlag = "--data-dir" - MysqlDataDirFlag = "--datadir" - MysqlInitializeInsecureFlag = "--initialize-insecure" - cpuProfileFilename = "cpu.pprof" -) - -var ( - ErrTestNameNotDefined = errors.New("test name not defined") - ErrNoServersDefined = errors.New("servers not defined") - ErrTooManyServersDefined = errors.New("too many servers defined, two max") - ErrUnsupportedConnectionProtocol = errors.New("unsupported connection protocol") -) - -var defaultSysbenchParams = []string{ - "--db-ps-mode=disable", - "--rand-type=uniform", -} - -var defaultDoltServerParams = []string{"sql-server"} - -var defaultSysbenchTests = []*ConfigTest{ - NewConfigTest("oltp_read_only", []string{}, false), - NewConfigTest("oltp_insert", []string{}, false), - NewConfigTest("bulk_insert", []string{}, false), - NewConfigTest("oltp_point_select", []string{}, false), - NewConfigTest("select_random_points", []string{}, false), - NewConfigTest("select_random_ranges", []string{}, false), - NewConfigTest("oltp_write_only", []string{}, false), - NewConfigTest("oltp_read_write", []string{}, false), - NewConfigTest("oltp_update_index", []string{}, false), - NewConfigTest("oltp_update_non_index", []string{}, false), -} - -var defaultDoltLuaScripts = map[string]string{ - "covering_index_scan.lua": "covering_index_scan.lua", - "groupby_scan.lua": "groupby_scan.lua", - "index_join.lua": "index_join.lua", - "index_join_scan.lua": "index_join_scan.lua", - "index_scan.lua": "index_scan.lua", - "oltp_delete_insert.lua": "oltp_delete_insert.lua", - "table_scan.lua": "table_scan.lua", - "types_delete_insert.lua": "types_delete_insert.lua", - "types_table_scan.lua": "types_table_scan.lua", -} - -// todo: check expressions need to be supported in doltgres for these -// todo: postgres does not have geometry types also -var defaultDoltgresLuaScripts = map[string]string{ - //"covering_index_scan_postgres.lua": "covering_index_scan_postgres.lua", - //"groupby_scan_postgres.lua": "groupby_scan_postgres.lua", - //"index_join_postgres.lua": "index_join_postgres.lua", - //"index_join_scan_postgres.lua": "index_join_scan_postgres.lua", - //"index_scan_postgres.lua": "index_scan_postgres.lua", - //"oltp_delete_insert_postgres.lua": "oltp_delete_insert_postgres.lua", - //"table_scan_postgres.lua": "table_scan_postgres.lua", - //"types_delete_insert_postgres.lua": "types_delete_insert_postgres.lua", - //"types_table_scan_postgres.lua": "types_table_scan_postgres.lua", -} - -type ServerType string - -// Test is a single sysbench test -type Test struct { - id string - - // Name is the test name - Name string - - // Params are the parameters passed to sysbench - Params []string - - // FromScript indicates if this test is from a lua script - FromScript bool -} - -// Prepare returns a test's args for sysbench's prepare step -func (t *Test) Prepare() []string { - return withCommand(t.Params, "prepare") -} - -// Run returns a test's args for sysbench's run step -func (t *Test) Run() []string { - return withCommand(t.Params, "run") -} - -// Cleanup returns a test's args for sysbench's cleanup step -func (t *Test) Cleanup() []string { - return withCommand(t.Params, "cleanup") -} - -func withCommand(params []string, command string) []string { - c := make([]string, 0) - c = append(c, params...) - return append(c, command) -} - -// ConfigTest provides users a way to define a test for multiple tablesizes -type ConfigTest struct { - // Name is the test name - Name string - - // N is the number of times a test should run - N int - - // Options are additional sysbench test options a user can supply to run with this test - Options []string - - // FromScript is a boolean indicating that this test is from a lua script - FromScript bool -} - -// NewConfigTest returns a ConfigTest containing the supplied args -func NewConfigTest(name string, opts []string, fromScript bool) *ConfigTest { - options := make([]string, 0) - options = append(options, opts...) - return &ConfigTest{ - Name: name, - N: 1, - Options: options, - FromScript: fromScript, - } -} - -// GetTests returns a slice of Tests -func (ct *ConfigTest) GetTests(serverConfig *ServerConfig, testIdFunc func() string) ([]*Test, error) { - if ct.Name == "" { - return nil, ErrTestNameNotDefined - } - if ct.N < 1 { - ct.N = 1 - } - - params := fromConfigTestParams(ct, serverConfig) - tests := make([]*Test, 0) - - var idFunc func() string - if testIdFunc == nil { - idFunc = func() string { - return uuid.New().String() - } - } else { - idFunc = testIdFunc - } - - for i := 0; i < ct.N; i++ { - p := make([]string, len(params)) - copy(p, params) - tests = append(tests, &Test{ - id: idFunc(), - Name: ct.Name, - Params: p, - FromScript: ct.FromScript, - }) - } - return tests, nil -} - -// fromConfigTestParams returns params formatted for sysbench: -// `sysbench [options]... [testname] [command]` -func fromConfigTestParams(ct *ConfigTest, serverConfig *ServerConfig) []string { - params := make([]string, 0) - params = append(params, defaultSysbenchParams...) - if serverConfig.Server == MySql || serverConfig.Server == Dolt { - params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) - params = append(params, "--db-driver=mysql") - params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host)) - if serverConfig.Port != 0 { - params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port)) - } - } else if serverConfig.Server == Doltgres || serverConfig.Server == Postgres { - params = append(params, "--db-driver=pgsql") - params = append(params, fmt.Sprintf("--pgsql-db=%s", dbName)) - params = append(params, fmt.Sprintf("--pgsql-host=%s", serverConfig.Host)) - if serverConfig.Port != 0 { - params = append(params, fmt.Sprintf("--pgsql-port=%d", serverConfig.Port)) - } - } - - // handle sysbench user for local mysql server - if serverConfig.Server == MySql && serverConfig.Host == defaultHost { - params = append(params, "--mysql-user=sysbench") - params = append(params, fmt.Sprintf("--mysql-password=%s", sysbenchPassLocal)) - } else if serverConfig.Server == Dolt { - params = append(params, "--mysql-user=root") - } else if serverConfig.Server == Doltgres { - params = append(params, "--pgsql-user=doltgres") - } else if serverConfig.Server == Postgres { - params = append(params, "--pgsql-user=postgres") - } - - params = append(params, ct.Options...) - params = append(params, ct.Name) - return params -} - -// ServerConfig is the configuration for a server to test against -type ServerConfig struct { - // Id is a unique id for this servers benchmarking - Id string - - // Host is the server host - Host string - - // Port is the server port - Port int - - // Server is the type of server - Server ServerType - - // Version is the server version - Version string - - // ResultsFormat is the format the results should be written in - ResultsFormat string - - // ServerExec is the path to a server executable - ServerExec string - - // InitExec is the path to the server init db executable - InitExec string - - // ServerUser is the user account that should start the server - ServerUser string - - // SkipLogBin will skip bin logging - SkipLogBin bool - - // ServerArgs are the args used to start a server - ServerArgs []string - - // ConnectionProtocol defines the protocol for connecting to the server - ConnectionProtocol string - - // Socket is the path to the server socket - Socket string - - // ServerProfile specifies the golang profile to take of a Dolt server - ServerProfile string - - // ProfilePath path to directory where server profile will be written - ProfilePath string -} - -func (sc *ServerConfig) GetId() string { - if sc.Id == "" { - sc.Id = uuid.New().String() - } - return sc.Id -} - -// GetServerArgs returns the args used to start a server -func (sc *ServerConfig) GetServerArgs() ([]string, error) { - params := make([]string, 0) - - if sc.Server == Dolt { - params = append(params, defaultDoltServerParams...) - } else if sc.Server == MySql { - if sc.ServerUser != "" { - params = append(params, fmt.Sprintf("%s=%s", userFlag, sc.ServerUser)) - } - if sc.SkipLogBin { - params = append(params, skipBinLogFlag) - } - } - - if sc.Server == Dolt || sc.Server == Doltgres { - params = append(params, fmt.Sprintf("%s=%s", hostFlag, sc.Host)) - } - if sc.Port != 0 { - params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port)) - } - - params = append(params, sc.ServerArgs...) - return params, nil -} - -// Config is the configuration for a benchmarking run -type Config struct { - // Runs is the number of times to run all tests - Runs int - // RuntimeOS is the platform the benchmarks ran on - RuntimeOS string - // RuntimeGoArch is the runtime architecture - RuntimeGoArch string - // Servers are the servers to benchmark - Servers []*ServerConfig - // Tests are the tests to run. If no tests are provided, - // the default tests will be used - Tests []*ConfigTest - // TestOptions a list of sysbench test options to apply to all tests - TestOptions []string - // ScriptDir is a path to a directory of lua scripts - ScriptDir string - // InitBigRepo downloads a database with existing chunks and commits - InitBigRepo bool - // NomsBinFormat specifies the NomsBinFormat - NomsBinFormat string -} - -// NewConfig returns a new Config -func NewConfig() *Config { - return &Config{ - Servers: make([]*ServerConfig, 0), - } -} - -// Validate checks the config for the required fields and sets defaults -// where necessary -func (c *Config) Validate() error { - if len(c.Servers) < 1 { - return ErrNoServersDefined - } - if len(c.Servers) > 2 { - return ErrTooManyServersDefined - } - err := c.setDefaults() - if err != nil { - return err - } - return c.validateServerConfigs() -} - -// validateServerConfigs ensures the ServerConfigs are valid -func (c *Config) validateServerConfigs() error { - portMap := make(map[int]ServerType) - for _, s := range c.Servers { - if s.Server != Dolt && s.Server != MySql && s.Server != Doltgres && s.Server != Postgres { - return fmt.Errorf("unsupported server type: %s", s.Server) - } - - err := ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat) - if err != nil { - return err - } - - if s.Server == MySql { - err = CheckProtocol(s.ConnectionProtocol) - if err != nil { - return err - } - } - - if s.Host == "" { - s.Host = defaultHost - } - - portMap, err = CheckUpdatePortMap(s, portMap) - if err != nil { - return err - } - - err = CheckExec(s.ServerExec, "server exec") - if err != nil { - return err - } - - if s.Server == Postgres { - err = CheckExec(s.InitExec, "initdb exec") - if err != nil { - return err - } - } - - if s.Server != Dolt && s.ServerProfile != "" { - return fmt.Errorf("profiling can only be done against a dolt server") - } - - if s.Server == Dolt && s.ServerProfile != "" { - if s.ServerProfile != cpuProfile { - return fmt.Errorf("unsupported server profile: %s", s.ServerProfile) - } - if s.ProfilePath == "" { - cwd, err := os.Getwd() - if err != nil { - return err - } - s.ProfilePath = cwd - } - } - } - - return nil -} - -func (c *Config) Contains(st ServerType) bool { - for _, s := range c.Servers { - if s.Server == st { - return true - } - } - return false -} - -func ValidateRequiredFields(server, version, format string) error { - if server == "" { - return getMustSupplyError("server") - } - if version == "" { - return getMustSupplyError("version") - } - if format == "" { - return getMustSupplyError("results format") - } - return nil -} - -// setDefaults sets defaults on the Config -func (c *Config) setDefaults() error { - if c.RuntimeOS == "" { - c.RuntimeOS = runtime.GOOS - } - if c.RuntimeGoArch == "" { - c.RuntimeGoArch = runtime.GOARCH - } - if len(c.Tests) < 1 { - fmt.Printf("Preparing to benchmark against default tests\n") - if c.ScriptDir != "" { - abs, err := filepath.Abs(c.ScriptDir) - if err != nil { - return err - } - if _, err := os.Stat(abs); os.IsNotExist(err) { - return fmt.Errorf("script dir not found: %s", abs) - } - c.ScriptDir = abs - } - tests, err := getDefaultTests(c) - if err != nil { - return err - } - c.Tests = tests - } - if c.Runs < 1 { - c.Runs = 1 - } - return nil -} - -// CheckUpdatePortMap returns an error if multiple servers have specified the same port -func CheckUpdatePortMap(serverConfig *ServerConfig, portMap map[int]ServerType) (map[int]ServerType, error) { - if serverConfig.Port == 0 { - serverConfig.Port = defaultPort - } - srv, ok := portMap[serverConfig.Port] - if ok && srv != serverConfig.Server { - return nil, fmt.Errorf("servers have port conflict on port: %d\n", serverConfig.Port) - } - if !ok { - portMap[serverConfig.Port] = serverConfig.Server - } - return portMap, nil -} - -// CheckExec verifies the binary exists -func CheckExec(path, messageIfMissing string) error { - if path == "" { - return getMustSupplyError(messageIfMissing) - } - abs, err := filepath.Abs(path) - if err != nil { - return err - } - if _, err := os.Stat(abs); os.IsNotExist(err) { - return fmt.Errorf("exec not found: %s", abs) - } - return nil -} - -// CheckProtocol ensures the given protocol is supported -func CheckProtocol(protocol string) error { - if protocol == "" { - return getMustSupplyError("connection protocol") - } - if protocol == tcpProtocol || protocol == unixProtocol { - return nil - } - return ErrUnsupportedConnectionProtocol -} - -// GetTests returns a slice of Tests created from the -// defined ServerConfig.Tests -func GetTests(config *Config, serverConfig *ServerConfig, testIdFunc func() string) ([]*Test, error) { - flattened := make([]*Test, 0) - for _, t := range config.Tests { - if len(config.TestOptions) > 0 { - t.Options = append(t.Options, config.TestOptions...) - } - tests, err := t.GetTests(serverConfig, testIdFunc) - if err != nil { - return nil, err - } - flattened = append(flattened, tests...) - } - return flattened, nil -} - -// FromFileConfig returns a validated Config based on the config file at the configPath -func FromFileConfig(configPath string) (*Config, error) { - data, err := os.ReadFile(configPath) - if err != nil { - return nil, err - } - - config := NewConfig() - err = json.Unmarshal(data, config) - if err != nil { - return nil, err - } - - return config, nil -} - -func getMustSupplyError(name string) error { - return fmt.Errorf("Must supply %s", name) -} - -func getDefaultTests(config *Config) ([]*ConfigTest, error) { - defaultTests := make([]*ConfigTest, 0) - defaultTests = append(defaultTests, defaultSysbenchTests...) - if config.ScriptDir != "" { - var luaScriptTests []*ConfigTest - var err error - if !config.Contains(Doltgres) && !config.Contains(Postgres) { - luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltLuaScripts) - } else { - luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltgresLuaScripts) - } - if err != nil { - return nil, err - } - defaultTests = append(defaultTests, luaScriptTests...) - } - return defaultTests, nil -} - -func getLuaScriptTestsFromDir(dir string, toInclude map[string]string) ([]*ConfigTest, error) { - luaScripts := make([]*ConfigTest, 0) - abs, err := filepath.Abs(dir) - if err != nil { - return nil, err - } - err = filepath.Walk(abs, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } +import "context" - file := filepath.Base(path) - if _, ok := toInclude[file]; ok { - luaScripts = append(luaScripts, NewConfigTest(path, []string{}, true)) - } - return nil - }) - if err != nil { - return nil, err - } - return luaScripts, nil +type Config interface { + Validate(ctx context.Context) error + ContainsServerOfType(server ServerType) bool } diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go index 30430416243..58fcdbecd6b 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -31,13 +31,13 @@ var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } type doltBenchmarkerImpl struct { dir string // cwd - config *Config - serverConfig *ServerConfig + config *sysbenchRunnerConfigImpl + serverConfig *doltServerConfigImpl } var _ Benchmarker = &doltBenchmarkerImpl{} -func NewDoltBenchmarker(dir string, config *Config, serverConfig *ServerConfig) *doltBenchmarkerImpl { +func NewDoltBenchmarker(dir string, config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl) *doltBenchmarkerImpl { return &doltBenchmarkerImpl{ dir: dir, config: config, diff --git a/go/performance/utils/sysbench_runner/dolt_server_config.go b/go/performance/utils/sysbench_runner/dolt_server_config.go new file mode 100644 index 00000000000..148f6712069 --- /dev/null +++ b/go/performance/utils/sysbench_runner/dolt_server_config.go @@ -0,0 +1,141 @@ +package sysbench_runner + +import ( + "fmt" + "github.com/google/uuid" + "os" +) + +const ( + CpuServerProfile = "cpu" +) + +type ServerProfile string + +type doltServerConfigImpl struct { + // Id is a unique id for this servers benchmarking + Id string + + // Host is the server host + Host string + + // Port is the server port + Port int + + // Version is the server version + Version string + + // ResultsFormat is the format the results should be written in + ResultsFormat string + + // ServerExec is the path to a server executable + ServerExec string + + // ServerUser is the user account that should start the server + ServerUser string + + // ServerArgs are the args used to start a server + ServerArgs []string + + // ServerProfile specifies the golang profile to take of a Dolt server + ServerProfile ServerProfile + + // ProfilePath path to directory where server profile will be written + ProfilePath string +} + +var _ ProfilingServerConfig = &doltServerConfigImpl{} + +func NewDoltServerConfig(version, serverExec, serverUser, host, resultsFormat, profilePath string, serverProfile ServerProfile, port int, serverArgs []string) *doltServerConfigImpl { + return &doltServerConfigImpl{ + Id: uuid.New().String(), + Host: host, + Port: port, + Version: version, + ResultsFormat: resultsFormat, + ServerExec: serverExec, + ServerUser: serverUser, + ServerArgs: serverArgs, + ServerProfile: serverProfile, + ProfilePath: profilePath, + } +} + +func (sc *doltServerConfigImpl) GetProfilePath() string { + return sc.ProfilePath +} + +func (sc *doltServerConfigImpl) GetServerProfile() ServerProfile { + return sc.ServerProfile +} + +func (sc *doltServerConfigImpl) GetServerType() ServerType { + return Dolt +} + +func (sc *doltServerConfigImpl) GetServerExec() string { + return sc.ServerExec +} + +// GetServerArgs returns the args used to start a server +func (sc *doltServerConfigImpl) GetServerArgs() ([]string, error) { + params := make([]string, 0) + params = append(params, defaultDoltServerParams...) + if sc.Host != "" { + params = append(params, fmt.Sprintf("%s=%s", hostFlag, sc.Host)) + } + if sc.Port != 0 { + params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port)) + } + params = append(params, sc.ServerArgs...) + return params, nil +} + +func (sc *doltServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { + params := make([]string, 0) + params = append(params, defaultSysbenchParams...) + params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) + params = append(params, "--db-driver=mysql") + params = append(params, fmt.Sprintf("--mysql-host=%s", sc.Host)) + params = append(params, "--mysql-user=root") + if sc.Port != 0 { + params = append(params, fmt.Sprintf("--mysql-port=%d", sc.Port)) + } + params = append(params, testConfig.GetOptions()...) + params = append(params, testConfig.GetName()) + return params +} + +func (sc *doltServerConfigImpl) Validate() error { + if sc.Version == "" { + return getMustSupplyError("version") + } + if sc.ResultsFormat == "" { + return getMustSupplyError("results format") + } + if sc.ServerExec == "" { + return getMustSupplyError("server exec") + } + if sc.ServerProfile != "" { + if sc.ServerProfile != CpuServerProfile { + return fmt.Errorf("unsupported server profile: %s", sc.ServerProfile) + } + } + return CheckExec(sc.ServerExec, "server exec") +} + +func (sc *doltServerConfigImpl) SetDefaults() error { + if sc.Host == "" { + sc.Host = defaultHost + } + if sc.ServerProfile != "" { + if sc.ProfilePath == "" { + cwd, err := os.Getwd() + if err != nil { + return err + } + sc.ProfilePath = cwd + } + } + return nil +} diff --git a/go/performance/utils/sysbench_runner/dolt_tpcc.go b/go/performance/utils/sysbench_runner/dolt_tpcc.go index 92a5a07ac4b..9721418abe7 100644 --- a/go/performance/utils/sysbench_runner/dolt_tpcc.go +++ b/go/performance/utils/sysbench_runner/dolt_tpcc.go @@ -15,12 +15,12 @@ const ( type doltTpccBenchmarkerImpl struct { dir string // cwd config *TpccBenchmarkConfig - serverConfig *ServerConfig + serverConfig *doltServerConfigImpl } var _ Benchmarker = &doltTpccBenchmarkerImpl{} -func NewDoltTpccBenchmarker(dir string, config *TpccBenchmarkConfig, serverConfig *ServerConfig) *doltTpccBenchmarkerImpl { +func NewDoltTpccBenchmarker(dir string, config *TpccBenchmarkConfig, serverConfig *doltServerConfigImpl) *doltTpccBenchmarkerImpl { return &doltTpccBenchmarkerImpl{ dir: dir, config: config, diff --git a/go/performance/utils/sysbench_runner/doltgres.go b/go/performance/utils/sysbench_runner/doltgres.go index 9d962982e4a..7e2d250c5cd 100644 --- a/go/performance/utils/sysbench_runner/doltgres.go +++ b/go/performance/utils/sysbench_runner/doltgres.go @@ -19,13 +19,13 @@ const ( type doltgresBenchmarkerImpl struct { dir string // cwd - config *Config - serverConfig *ServerConfig + config *sysbenchRunnerConfigImpl + serverConfig *doltServerConfigImpl } var _ Benchmarker = &doltgresBenchmarkerImpl{} -func NewDoltgresBenchmarker(dir string, config *Config, serverConfig *ServerConfig) *doltgresBenchmarkerImpl { +func NewDoltgresBenchmarker(dir string, config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl) *doltgresBenchmarkerImpl { return &doltgresBenchmarkerImpl{ dir: dir, config: config, @@ -116,7 +116,7 @@ func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Result return } - var tests []*Test + var tests []*sysbenchTestImpl tests, err = GetTests(b.config, b.serverConfig, nil) if err != nil { return diff --git a/go/performance/utils/sysbench_runner/doltgres_server_config.go b/go/performance/utils/sysbench_runner/doltgres_server_config.go new file mode 100644 index 00000000000..e997c35e7ae --- /dev/null +++ b/go/performance/utils/sysbench_runner/doltgres_server_config.go @@ -0,0 +1,102 @@ +package sysbench_runner + +import ( + "fmt" + "github.com/google/uuid" +) + +type doltgresServerConfigImpl struct { + // Id is a unique id for this servers benchmarking + Id string + + // Host is the server host + Host string + + // Port is the server port + Port int + + // Version is the server version + Version string + + // ResultsFormat is the format the results should be written in + ResultsFormat string + + // ServerExec is the path to a server executable + ServerExec string + + // ServerUser is the user account that should start the server + ServerUser string + + // ServerArgs are the args used to start a server + ServerArgs []string +} + +var _ ServerConfig = &doltgresServerConfigImpl{} + +func NewDoltgresServerConfig(version, serverExec, serverUser, host, resultsFormat string, port int, serverArgs []string) *doltgresServerConfigImpl { + return &doltgresServerConfigImpl{ + Id: uuid.New().String(), + Host: host, + Port: port, + Version: version, + ResultsFormat: resultsFormat, + ServerExec: serverExec, + ServerUser: serverUser, + ServerArgs: serverArgs, + } +} + +func (sc *doltgresServerConfigImpl) GetServerType() ServerType { + return Doltgres +} + +func (sc *doltgresServerConfigImpl) GetServerExec() string { + return sc.ServerExec +} + +func (sc *doltgresServerConfigImpl) GetServerArgs() ([]string, error) { + params := make([]string, 0) + if sc.Host != "" { + params = append(params, fmt.Sprintf("%s=%s", hostFlag, sc.Host)) + } + if sc.Port != 0 { + params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port)) + } + params = append(params, sc.ServerArgs...) + return params, nil +} + +func (sc *doltgresServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { + params := make([]string, 0) + params = append(params, defaultSysbenchParams...) + params = append(params, "--db-driver=pgsql") + params = append(params, fmt.Sprintf("--pgsql-db=%s", dbName)) + params = append(params, fmt.Sprintf("--pgsql-host=%s", sc.Host)) + params = append(params, "--pgsql-user=doltgres") + if sc.Port != 0 { + params = append(params, fmt.Sprintf("--pgsql-port=%d", sc.Port)) + } + params = append(params, testConfig.GetOptions()...) + params = append(params, testConfig.GetName()) + return params +} + +func (sc *doltgresServerConfigImpl) Validate() error { + if sc.Version == "" { + return getMustSupplyError("version") + } + if sc.ResultsFormat == "" { + return getMustSupplyError("results format") + } + if sc.ServerExec == "" { + return getMustSupplyError("server exec") + } + return CheckExec(sc.ServerExec, "server exec") +} + +func (sc *doltgresServerConfigImpl) SetDefaults() error { + if sc.Host == "" { + sc.Host = defaultHost + } + return nil +} diff --git a/go/performance/utils/sysbench_runner/mysql.go b/go/performance/utils/sysbench_runner/mysql.go index 05d3f64a218..b962feefe64 100644 --- a/go/performance/utils/sysbench_runner/mysql.go +++ b/go/performance/utils/sysbench_runner/mysql.go @@ -25,13 +25,13 @@ const ( type mysqlBenchmarkerImpl struct { dir string // cwd - config *Config - serverConfig *ServerConfig + config *sysbenchRunnerConfigImpl + serverConfig *doltServerConfigImpl } var _ Benchmarker = &mysqlBenchmarkerImpl{} -func NewMysqlBenchmarker(dir string, config *Config, serverConfig *ServerConfig) *mysqlBenchmarkerImpl { +func NewMysqlBenchmarker(dir string, config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl) *mysqlBenchmarkerImpl { return &mysqlBenchmarkerImpl{ dir: dir, config: config, diff --git a/go/performance/utils/sysbench_runner/mysql_server_config.go b/go/performance/utils/sysbench_runner/mysql_server_config.go new file mode 100644 index 00000000000..42e075affe6 --- /dev/null +++ b/go/performance/utils/sysbench_runner/mysql_server_config.go @@ -0,0 +1,122 @@ +package sysbench_runner + +import ( + "fmt" + "github.com/google/uuid" +) + +type mysqlServerConfigImpl struct { + // Id is a unique id for this servers benchmarking + Id string + + // Host is the server host + Host string + + // Port is the server port + Port int + + // Version is the server version + Version string + + // ResultsFormat is the format the results should be written in + ResultsFormat string + + // ServerExec is the path to a server executable + ServerExec string + + // ServerUser is the user account that should start the server + ServerUser string + + // SkipLogBin will skip bin logging + SkipLogBin bool + + // ServerArgs are the args used to start a server + ServerArgs []string + + // ConnectionProtocol defines the protocol for connecting to the server + ConnectionProtocol string + + // Socket is the path to the server socket + Socket string +} + +var _ ServerConfig = &mysqlServerConfigImpl{} + +func NewMysqlServerConfig(version, serverExec, serverUser, host, resultsFormat, protocol, socket string, port int, serverArgs []string, skipBinLog bool) *mysqlServerConfigImpl { + return &mysqlServerConfigImpl{ + Id: uuid.New().String(), + Host: host, + Port: port, + Version: version, + ResultsFormat: resultsFormat, + ServerExec: serverExec, + ServerUser: serverUser, + SkipLogBin: skipBinLog, + ServerArgs: serverArgs, + ConnectionProtocol: protocol, + Socket: socket, + } +} + +func (sc *mysqlServerConfigImpl) GetServerExec() string { + return sc.ServerExec +} + +func (sc *mysqlServerConfigImpl) GetServerType() ServerType { + return MySql +} + +func (sc *mysqlServerConfigImpl) GetServerArgs() ([]string, error) { + params := make([]string, 0) + if sc.ServerUser != "" { + params = append(params, fmt.Sprintf("%s=%s", userFlag, sc.ServerUser)) + } + if sc.SkipLogBin { + params = append(params, skipBinLogFlag) + } + if sc.Port != 0 { + params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port)) + } + params = append(params, sc.ServerArgs...) + return params, nil +} + +func (sc *mysqlServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { + params := make([]string, 0) + params = append(params, defaultSysbenchParams...) + params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) + params = append(params, "--db-driver=mysql") + params = append(params, fmt.Sprintf("--mysql-host=%s", sc.Host)) + if sc.Port != 0 { + params = append(params, fmt.Sprintf("--mysql-port=%d", sc.Port)) + } + params = append(params, "--mysql-user=sysbench") + params = append(params, fmt.Sprintf("--mysql-password=%s", sysbenchPassLocal)) + params = append(params, testConfig.GetOptions()...) + params = append(params, testConfig.GetName()) + return params +} + +func (sc *mysqlServerConfigImpl) Validate() error { + if sc.Version == "" { + return getMustSupplyError("version") + } + if sc.ResultsFormat == "" { + return getMustSupplyError("results format") + } + if sc.ServerExec == "" { + return getMustSupplyError("server exec") + } + err := CheckProtocol(sc.ConnectionProtocol) + if err != nil { + return err + } + return CheckExec(sc.ServerExec, "server exec") +} + +func (sc *mysqlServerConfigImpl) SetDefaults() error { + if sc.Host == "" { + sc.Host = defaultHost + } + return nil +} diff --git a/go/performance/utils/sysbench_runner/mysql_tpcc.go b/go/performance/utils/sysbench_runner/mysql_tpcc.go index 61ff4232e22..87fb14c9602 100644 --- a/go/performance/utils/sysbench_runner/mysql_tpcc.go +++ b/go/performance/utils/sysbench_runner/mysql_tpcc.go @@ -10,12 +10,12 @@ import ( type mysqlTpccBenchmarkerImpl struct { dir string // cwd config *TpccBenchmarkConfig - serverConfig *ServerConfig + serverConfig *doltServerConfigImpl } var _ Benchmarker = &mysqlTpccBenchmarkerImpl{} -func NewMysqlTpccBenchmarker(dir string, config *TpccBenchmarkConfig, serverConfig *ServerConfig) *mysqlTpccBenchmarkerImpl { +func NewMysqlTpccBenchmarker(dir string, config *TpccBenchmarkConfig, serverConfig *doltServerConfigImpl) *mysqlTpccBenchmarkerImpl { return &mysqlTpccBenchmarkerImpl{ dir: dir, config: config, diff --git a/go/performance/utils/sysbench_runner/postgres.go b/go/performance/utils/sysbench_runner/postgres.go index fa1416d334f..264a70adec5 100644 --- a/go/performance/utils/sysbench_runner/postgres.go +++ b/go/performance/utils/sysbench_runner/postgres.go @@ -25,13 +25,13 @@ const ( type postgresBenchmarkerImpl struct { dir string // cwd - config *Config - serverConfig *ServerConfig + config *sysbenchRunnerConfigImpl + serverConfig *doltServerConfigImpl } var _ Benchmarker = &postgresBenchmarkerImpl{} -func NewPostgresBenchmarker(dir string, config *Config, serverConfig *ServerConfig) *postgresBenchmarkerImpl { +func NewPostgresBenchmarker(dir string, config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl) *postgresBenchmarkerImpl { return &postgresBenchmarkerImpl{ dir: dir, config: config, @@ -124,7 +124,7 @@ func (b *postgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Result return } - var tests []*Test + var tests []*sysbenchTestImpl tests, err = GetTests(b.config, b.serverConfig, nil) if err != nil { return diff --git a/go/performance/utils/sysbench_runner/postgres_server_config.go b/go/performance/utils/sysbench_runner/postgres_server_config.go new file mode 100644 index 00000000000..77cbbe153f5 --- /dev/null +++ b/go/performance/utils/sysbench_runner/postgres_server_config.go @@ -0,0 +1,111 @@ +package sysbench_runner + +import ( + "fmt" + "github.com/google/uuid" +) + +type postgresServerConfigImpl struct { + // Id is a unique id for this servers benchmarking + Id string + + // Host is the server host + Host string + + // Port is the server port + Port int + + // Version is the server version + Version string + + // ResultsFormat is the format the results should be written in + ResultsFormat string + + // ServerExec is the path to a server executable + ServerExec string + + // InitExec is the path to the server init db executable + InitExec string + + // ServerUser is the user account that should start the server + ServerUser string + + // ServerArgs are the args used to start a server + ServerArgs []string +} + +var _ InitServerConfig = &postgresServerConfigImpl{} + +func NewPostgresServerConfig(version, serverExec, initDbExec, serverUser, host, resultsFormat string, port int, serverArgs []string) *postgresServerConfigImpl { + return &postgresServerConfigImpl{ + Id: uuid.New().String(), + Host: host, + Port: port, + Version: version, + ResultsFormat: resultsFormat, + ServerExec: serverExec, + InitExec: initDbExec, + ServerUser: serverUser, + ServerArgs: serverArgs, + } +} + +func (sc *postgresServerConfigImpl) GetServerExec() string { + return sc.ServerExec +} + +func (sc *postgresServerConfigImpl) GetInitDbExec() string { + return sc.InitExec +} + +func (sc *postgresServerConfigImpl) GetServerType() ServerType { + return Postgres +} + +func (sc *postgresServerConfigImpl) GetServerArgs() ([]string, error) { + params := make([]string, 0) + if sc.Port != 0 { + params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port)) + } + params = append(params, sc.ServerArgs...) + return params, nil +} + +func (sc *postgresServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { + params := make([]string, 0) + params = append(params, defaultSysbenchParams...) + params = append(params, "--db-driver=pgsql") + params = append(params, fmt.Sprintf("--pgsql-db=%s", dbName)) + params = append(params, fmt.Sprintf("--pgsql-host=%s", sc.Host)) + params = append(params, "--pgsql-user=postgres") + if sc.Port != 0 { + params = append(params, fmt.Sprintf("--pgsql-port=%d", sc.Port)) + } + params = append(params, testConfig.GetOptions()...) + params = append(params, testConfig.GetName()) + return params +} + +func (sc *postgresServerConfigImpl) Validate() error { + if sc.Version == "" { + return getMustSupplyError("version") + } + if sc.ResultsFormat == "" { + return getMustSupplyError("results format") + } + if sc.ServerExec == "" { + return getMustSupplyError("server exec") + } + err := CheckExec(sc.ServerExec, "server exec") + if err != nil { + return err + } + return CheckExec(sc.InitExec, "initdb exec") +} + +func (sc *postgresServerConfigImpl) SetDefaults() error { + if sc.Host == "" { + sc.Host = defaultHost + } + return nil +} diff --git a/go/performance/utils/sysbench_runner/profile.go b/go/performance/utils/sysbench_runner/profile.go index 877ed41ed29..ae0cdbb83fb 100644 --- a/go/performance/utils/sysbench_runner/profile.go +++ b/go/performance/utils/sysbench_runner/profile.go @@ -14,13 +14,13 @@ type Profiler interface { type doltProfilerImpl struct { dir string // cwd - config *Config - serverConfig *ServerConfig + config *sysbenchRunnerConfigImpl + serverConfig *doltServerConfigImpl } var _ Profiler = &doltProfilerImpl{} -func NewDoltProfiler(dir string, config *Config, serverConfig *ServerConfig) *doltProfilerImpl { +func NewDoltProfiler(dir string, config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl) *doltProfilerImpl { return &doltProfilerImpl{ dir: dir, config: config, diff --git a/go/performance/utils/sysbench_runner/results.go b/go/performance/utils/sysbench_runner/results.go index bb30d1662f0..99029ddff68 100644 --- a/go/performance/utils/sysbench_runner/results.go +++ b/go/performance/utils/sysbench_runner/results.go @@ -152,45 +152,6 @@ func (r *Result) Stamp(stampFunc func() string) { r.CreatedAt = stampFunc() } -// FromConfigsNewResult returns a new result with some fields set based on the provided configs -func FromConfigsNewResult(config *Config, serverConfig *ServerConfig, t *Test, suiteId string, idFunc func() string) (*Result, error) { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return nil, err - } - - var getId func() string - if idFunc == nil { - getId = func() string { - return uuid.New().String() - } - } else { - getId = idFunc - } - - var name string - if t.FromScript { - base := filepath.Base(t.Name) - ext := filepath.Ext(base) - name = strings.TrimSuffix(base, ext) - } else { - name = t.Name - } - - return &Result{ - Id: getId(), - SuiteId: suiteId, - TestId: t.id, - RuntimeOS: config.RuntimeOS, - RuntimeGoArch: config.RuntimeGoArch, - ServerName: string(serverConfig.Server), - ServerVersion: serverConfig.Version, - ServerParams: strings.Join(serverParams, " "), - TestName: name, - TestParams: strings.Join(t.Params, " "), - }, nil -} - func NewResult(server ServerType, version, testName, testId, suiteId, runtimeOs, runtimeGoArch string, serverParams, testParams []string, idFunc func() string, fromScript bool) *Result { var getId func() string if idFunc == nil { @@ -224,33 +185,6 @@ func NewResult(server ServerType, version, testName, testId, suiteId, runtimeOs, } } -// FromOutputResult accepts raw sysbench run output and returns the Result -func FromOutputResult(output []byte, config *Config, serverConfig *ServerConfig, test *Test, suiteId string, idFunc func() string) (*Result, error) { - result, err := FromConfigsNewResult(config, serverConfig, test, suiteId, idFunc) - if err != nil { - return nil, err - } - lines := strings.Split(string(output), "\n") - var process bool - for _, l := range lines { - trimmed := strings.TrimSpace(l) - if trimmed == "" { - continue - } - if strings.HasPrefix(trimmed, SqlStatsPrefix) { - process = true - continue - } - if process { - err := UpdateResult(result, trimmed) - if err != nil { - return result, err - } - } - } - return result, nil -} - // UpdateResult extracts the key and value from the given line and updates the given Result func UpdateResult(result *Result, line string) error { lineParts := strings.Split(line, ":") diff --git a/go/performance/utils/sysbench_runner/results_test.go b/go/performance/utils/sysbench_runner/results_test.go index b1e3cb5c345..76db1f40f16 100644 --- a/go/performance/utils/sysbench_runner/results_test.go +++ b/go/performance/utils/sysbench_runner/results_test.go @@ -266,26 +266,26 @@ func TestFromOutputResults(t *testing.T) { tests := []struct { description string output []byte - config *Config - serverConfig *ServerConfig - test *Test + config *sysbenchRunnerConfigImpl + serverConfig *doltServerConfigImpl + test *sysbenchTestImpl expectedResult *Result expectedError error }{ { description: "should parse output into result", output: []byte(sampleOutput1), - config: &Config{ + config: &sysbenchRunnerConfigImpl{ RuntimeOS: testOS, RuntimeGoArch: testGoArch, }, - serverConfig: &ServerConfig{ + serverConfig: &doltServerConfigImpl{ Host: "localhost", Server: ServerType(testServer), Version: testServerVersion, ServerExec: "test-exec", }, - test: &Test{ + test: &sysbenchTestImpl{ Name: testTestName, Params: []string{testTestParams}, }, @@ -317,9 +317,9 @@ func TestFromOutputResults(t *testing.T) { } for _, test := range tests { t.Run(test.description, func(t *testing.T) { - actual, err := FromOutputResult(test.output, test.config, test.serverConfig, test.test, testSuiteId, func() string { - return testId - }) + serverParams, err := test.serverConfig.GetServerArgs() + assert.NoError(t, err) + actual, err := OutputToResult(test.output, test.serverConfig.Server, test.serverConfig.Version, test.test.Name, test.test.id, testSuiteId, test.config.RuntimeOS, test.config.RuntimeGoArch, serverParams, test.test.Params, func() string { return testId }, test.test.FromScript) assert.Equal(t, test.expectedError, err) assert.Equal(t, test.expectedResult, actual) }) diff --git a/go/performance/utils/sysbench_runner/run.go b/go/performance/utils/sysbench_runner/run.go index 2d5dfd58142..551cd5939a3 100644 --- a/go/performance/utils/sysbench_runner/run.go +++ b/go/performance/utils/sysbench_runner/run.go @@ -22,7 +22,7 @@ import ( ) // Run runs sysbench runner -func Run(ctx context.Context, config *Config) error { +func Run(ctx context.Context, config *sysbenchRunnerConfigImpl) error { err := config.Validate() if err != nil { return err @@ -86,7 +86,7 @@ func sysbenchVersion(ctx context.Context) error { return sysbenchVersion.Run() } -func WriteResults(serverConfig *ServerConfig, results Results) error { +func WriteResults(serverConfig *doltServerConfigImpl, results Results) error { cwd, err := os.Getwd() if err != nil { return err diff --git a/go/performance/utils/sysbench_runner/run_test.go b/go/performance/utils/sysbench_runner/run_test.go index 645de9ffb00..206b8328535 100644 --- a/go/performance/utils/sysbench_runner/run_test.go +++ b/go/performance/utils/sysbench_runner/run_test.go @@ -30,10 +30,10 @@ func TestRunner(t *testing.T) { log.Fatal(err) } - conf := &Config{ + conf := &sysbenchRunnerConfigImpl{ Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_delete_insert"), //Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_update_non_index", "oltp_insert", "bulk_insert", "oltp_write_only", "oltp_delete"), - Servers: []*ServerConfig{ + Servers: []*doltServerConfigImpl{ { Id: "test", Server: Dolt, @@ -59,10 +59,10 @@ func TestRunner(t *testing.T) { } } -func selectTests(names ...string) []*ConfigTest { - tests := make([]*ConfigTest, len(names)) +func selectTests(names ...string) []*TestConfigImpl { + tests := make([]*TestConfigImpl, len(names)) for i := range names { - tests[i] = &ConfigTest{Name: names[i], FromScript: false} + tests[i] = &TestConfigImpl{Name: names[i], FromScript: false} } return tests } diff --git a/go/performance/utils/sysbench_runner/server.go b/go/performance/utils/sysbench_runner/server.go index 4439c8164bc..44dcc8a07c4 100644 --- a/go/performance/utils/sysbench_runner/server.go +++ b/go/performance/utils/sysbench_runner/server.go @@ -26,7 +26,7 @@ type Server interface { type doltServerImpl struct { dir string - serverConfig *ServerConfig + serverConfig *doltServerConfigImpl serverCtx context.Context serverCtxCancelFunc context.CancelFunc server *exec.Cmd @@ -37,7 +37,7 @@ type doltServerImpl struct { var _ Server = &doltServerImpl{} -func NewServer(ctx context.Context, dir string, serverConfig *ServerConfig, killSignal os.Signal, serverParams []string) *doltServerImpl { +func NewServer(ctx context.Context, dir string, serverConfig *doltServerConfigImpl, killSignal os.Signal, serverParams []string) *doltServerImpl { withKeyCtx, cancel := context.WithCancel(ctx) gServer, serverCtx := errgroup.WithContext(withKeyCtx) diff --git a/go/performance/utils/sysbench_runner/server_config.go b/go/performance/utils/sysbench_runner/server_config.go new file mode 100644 index 00000000000..d4b398f354d --- /dev/null +++ b/go/performance/utils/sysbench_runner/server_config.go @@ -0,0 +1,23 @@ +package sysbench_runner + +type ServerType string + +type ServerConfig interface { + GetServerExec() string + GetServerType() ServerType + GetServerArgs() ([]string, error) + GetTestingArgs(testConfig TestConfig) []string + Validate() error + SetDefaults() error +} + +type InitServerConfig interface { + ServerConfig + GetInitDbExec() string +} + +type ProfilingServerConfig interface { + ServerConfig + GetServerProfile() ServerProfile + GetProfilePath() string +} diff --git a/go/performance/utils/sysbench_runner/sysbench.go b/go/performance/utils/sysbench_runner/sysbench.go index 00be936a8eb..590146be9e1 100644 --- a/go/performance/utils/sysbench_runner/sysbench.go +++ b/go/performance/utils/sysbench_runner/sysbench.go @@ -17,20 +17,20 @@ const ( luaPath = "?.lua" ) -type sysbenchTestImpl struct { - test *Test - config *Config - serverConfig *ServerConfig +type sysbenchTesterImpl struct { + test *sysbenchTestImpl + config *sysbenchRunnerConfigImpl + serverConfig *doltServerConfigImpl serverParams []string stampFunc func() string idFunc func() string suiteId string } -var _ Tester = &sysbenchTestImpl{} +var _ Tester = &sysbenchTesterImpl{} -func NewSysbenchTester(config *Config, serverConfig *ServerConfig, test *Test, serverParams []string, stampFunc func() string) *sysbenchTestImpl { - return &sysbenchTestImpl{ +func NewSysbenchTester(config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl, test *sysbenchTestImpl, serverParams []string, stampFunc func() string) *sysbenchTesterImpl { + return &sysbenchTesterImpl{ config: config, serverParams: serverParams, serverConfig: serverConfig, @@ -40,7 +40,7 @@ func NewSysbenchTester(config *Config, serverConfig *ServerConfig, test *Test, s } } -func (t *sysbenchTestImpl) newResult() (*Result, error) { +func (t *sysbenchTesterImpl) newResult() (*Result, error) { serverParams, err := t.serverConfig.GetServerArgs() if err != nil { return nil, err @@ -78,11 +78,11 @@ func (t *sysbenchTestImpl) newResult() (*Result, error) { }, nil } -func (t *sysbenchTestImpl) outputToResult(output []byte) (*Result, error) { +func (t *sysbenchTesterImpl) outputToResult(output []byte) (*Result, error) { return OutputToResult(output, t.serverConfig.Server, t.serverConfig.Version, t.test.Name, t.test.id, t.suiteId, t.config.RuntimeOS, t.config.RuntimeGoArch, t.serverParams, t.test.Params, nil, t.test.FromScript) } -func (t *sysbenchTestImpl) prepare(ctx context.Context) error { +func (t *sysbenchTesterImpl) prepare(ctx context.Context) error { cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.Prepare()...) if t.test.FromScript { lp := filepath.Join(t.config.ScriptDir, luaPath) @@ -97,7 +97,7 @@ func (t *sysbenchTestImpl) prepare(ctx context.Context) error { return nil } -func (t *sysbenchTestImpl) run(ctx context.Context) (*Result, error) { +func (t *sysbenchTesterImpl) run(ctx context.Context) (*Result, error) { cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.Run()...) if t.test.FromScript { lp := filepath.Join(t.config.ScriptDir, luaPath) @@ -125,7 +125,7 @@ func (t *sysbenchTestImpl) run(ctx context.Context) (*Result, error) { return rs, nil } -func (t *sysbenchTestImpl) cleanup(ctx context.Context) error { +func (t *sysbenchTesterImpl) cleanup(ctx context.Context) error { cmd := ExecCommand(ctx, sysbenchCommand, t.test.Cleanup()...) if t.test.FromScript { lp := filepath.Join(t.config.ScriptDir, luaPath) @@ -135,7 +135,7 @@ func (t *sysbenchTestImpl) cleanup(ctx context.Context) error { return cmd.Run() } -func (t *sysbenchTestImpl) Test(ctx context.Context) (*Result, error) { +func (t *sysbenchTesterImpl) Test(ctx context.Context) (*Result, error) { err := t.prepare(ctx) if err != nil { return nil, err diff --git a/go/performance/utils/sysbench_runner/sysbench_config.go b/go/performance/utils/sysbench_runner/sysbench_config.go new file mode 100644 index 00000000000..9785443b2a8 --- /dev/null +++ b/go/performance/utils/sysbench_runner/sysbench_config.go @@ -0,0 +1,510 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" +) + +const ( + Dolt ServerType = "dolt" + Doltgres ServerType = "doltgres" + Postgres ServerType = "postgres" + MySql ServerType = "mysql" + + CsvFormat = "csv" + JsonFormat = "json" + + CsvExt = ".csv" + JsonExt = ".json" + + defaultHost = "127.0.0.1" + defaultPort = 3306 + + defaultMysqlSocket = "/var/run/mysqld/mysqld.sock" + + tcpProtocol = "tcp" + unixProtocol = "unix" + + sysbenchUsername = "sysbench" + sysbenchUserLocal = "'sysbench'@'localhost'" + sysbenchPassLocal = "sysbenchpass" + + userFlag = "--user" + hostFlag = "--host" + portFlag = "--port" + skipBinLogFlag = "--skip-log-bin" + profileFlag = "--prof" + profilePathFlag = "--prof-path" + cpuProfile = "cpu" + doltgresDataDirFlag = "--data-dir" + MysqlDataDirFlag = "--datadir" + MysqlInitializeInsecureFlag = "--initialize-insecure" + cpuProfileFilename = "cpu.pprof" +) + +var ( + ErrTestNameNotDefined = errors.New("test name not defined") + ErrNoServersDefined = errors.New("servers not defined") + ErrTooManyServersDefined = errors.New("too many servers defined, two max") + ErrUnsupportedConnectionProtocol = errors.New("unsupported connection protocol") +) + +var defaultSysbenchParams = []string{ + "--db-ps-mode=disable", + "--rand-type=uniform", +} + +var defaultDoltServerParams = []string{"sql-server"} + +var defaultSysbenchTests = []TestConfig{ + NewTestConfig("oltp_read_only", []string{}, false), + NewTestConfig("oltp_insert", []string{}, false), + NewTestConfig("bulk_insert", []string{}, false), + NewTestConfig("oltp_point_select", []string{}, false), + NewTestConfig("select_random_points", []string{}, false), + NewTestConfig("select_random_ranges", []string{}, false), + NewTestConfig("oltp_write_only", []string{}, false), + NewTestConfig("oltp_read_write", []string{}, false), + NewTestConfig("oltp_update_index", []string{}, false), + NewTestConfig("oltp_update_non_index", []string{}, false), +} + +var defaultDoltLuaScripts = map[string]string{ + "covering_index_scan.lua": "covering_index_scan.lua", + "groupby_scan.lua": "groupby_scan.lua", + "index_join.lua": "index_join.lua", + "index_join_scan.lua": "index_join_scan.lua", + "index_scan.lua": "index_scan.lua", + "oltp_delete_insert.lua": "oltp_delete_insert.lua", + "table_scan.lua": "table_scan.lua", + "types_delete_insert.lua": "types_delete_insert.lua", + "types_table_scan.lua": "types_table_scan.lua", +} + +// todo: check expressions need to be supported in doltgres for these +// todo: postgres does not have geometry types also +var defaultDoltgresLuaScripts = map[string]string{ + //"covering_index_scan_postgres.lua": "covering_index_scan_postgres.lua", + //"groupby_scan_postgres.lua": "groupby_scan_postgres.lua", + //"index_join_postgres.lua": "index_join_postgres.lua", + //"index_join_scan_postgres.lua": "index_join_scan_postgres.lua", + //"index_scan_postgres.lua": "index_scan_postgres.lua", + //"oltp_delete_insert_postgres.lua": "oltp_delete_insert_postgres.lua", + //"table_scan_postgres.lua": "table_scan_postgres.lua", + //"types_delete_insert_postgres.lua": "types_delete_insert_postgres.lua", + //"types_table_scan_postgres.lua": "types_table_scan_postgres.lua", +} + +//// TestConfigImpl provides users a way to define a test for multiple tablesizes +//type TestConfig struct { +// // Name is the test name +// Name string +// +// // N is the number of times a test should run +// N int +// +// // Options are additional sysbench test options a user can supply to run with this test +// Options []string +// +// // FromScript is a boolean indicating that this test is from a lua script +// FromScript bool +//} + +//// NewConfigTest returns a TestConfigImpl containing the supplied args +//func NewConfigTest(name string, opts []string, fromScript bool) *TestConfig { +// options := make([]string, 0) +// options = append(options, opts...) +// return &TestConfig{ +// Name: name, +// N: 1, +// Options: options, +// FromScript: fromScript, +// } +//} + +// GetTests returns a slice of Tests +//func (ct *TestConfig) GetTests(serverConfig *doltServerConfigImpl, testIdFunc func() string) ([]*sysbenchTestImpl, error) { +// if ct.Name == "" { +// return nil, ErrTestNameNotDefined +// } +// if ct.N < 1 { +// ct.N = 1 +// } +// +// params := fromConfigTestParams(ct, serverConfig) +// tests := make([]*sysbenchTestImpl, 0) +// +// var idFunc func() string +// if testIdFunc == nil { +// idFunc = func() string { +// return uuid.New().String() +// } +// } else { +// idFunc = testIdFunc +// } +// +// for i := 0; i < ct.N; i++ { +// p := make([]string, len(params)) +// copy(p, params) +// tests = append(tests, &sysbenchTestImpl{ +// id: idFunc(), +// Name: ct.Name, +// Params: p, +// FromScript: ct.FromScript, +// }) +// } +// return tests, nil +//} + +//// fromConfigTestParams returns params formatted for sysbench: +//// `sysbench [options]... [testname] [command]` +//func fromConfigTestParams(ct *TestConfig, serverConfig *doltServerConfigImpl) []string { +// params := make([]string, 0) +// params = append(params, defaultSysbenchParams...) +// if serverConfig.Server == MySql || serverConfig.Server == Dolt { +// params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) +// params = append(params, "--db-driver=mysql") +// params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host)) +// if serverConfig.Port != 0 { +// params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port)) +// } +// } else if serverConfig.Server == Doltgres || serverConfig.Server == Postgres { +// params = append(params, "--db-driver=pgsql") +// params = append(params, fmt.Sprintf("--pgsql-db=%s", dbName)) +// params = append(params, fmt.Sprintf("--pgsql-host=%s", serverConfig.Host)) +// if serverConfig.Port != 0 { +// params = append(params, fmt.Sprintf("--pgsql-port=%d", serverConfig.Port)) +// } +// } +// +// // handle sysbench user for local mysql server +// if serverConfig.Server == MySql && serverConfig.Host == defaultHost { +// params = append(params, "--mysql-user=sysbench") +// params = append(params, fmt.Sprintf("--mysql-password=%s", sysbenchPassLocal)) +// } else if serverConfig.Server == Dolt { +// params = append(params, "--mysql-user=root") +// } else if serverConfig.Server == Doltgres { +// params = append(params, "--pgsql-user=doltgres") +// } else if serverConfig.Server == Postgres { +// params = append(params, "--pgsql-user=postgres") +// } +// +// params = append(params, ct.Options...) +// params = append(params, ct.Name) +// return params +//} + +// sysbenchRunnerConfigImpl is the configuration for a benchmarking run +type sysbenchRunnerConfigImpl struct { + // Runs is the number of times to run all tests + Runs int + // RuntimeOS is the platform the benchmarks ran on + RuntimeOS string + // RuntimeGoArch is the runtime architecture + RuntimeGoArch string + // Servers are the servers to benchmark + Servers []ServerConfig + // Tests are the tests to run. If no tests are provided, + // the default tests will be used + Tests []TestConfig + // TestOptions a list of sysbench test options to apply to all tests + TestOptions []string + // ScriptDir is a path to a directory of lua scripts + ScriptDir string + // InitBigRepo downloads a database with existing chunks and commits + InitBigRepo bool + // NomsBinFormat specifies the NomsBinFormat + NomsBinFormat string +} + +var _ Config = &sysbenchRunnerConfigImpl{} + +// NewRunnerConfig returns a new sysbenchRunnerConfigImpl +func NewRunnerConfig() *sysbenchRunnerConfigImpl { + return &sysbenchRunnerConfigImpl{ + Servers: make([]ServerConfig, 0), + } +} + +// Validate checks the config for the required fields and sets defaults +// where necessary +func (c *sysbenchRunnerConfigImpl) Validate(ctx context.Context) error { + if len(c.Servers) < 1 { + return ErrNoServersDefined + } + if len(c.Servers) > 2 { + return ErrTooManyServersDefined + } + err := c.setDefaults() + if err != nil { + return err + } + return c.validateServerConfigs() +} + +// validateServerConfigs ensures the ServerConfigs are valid +func (c *sysbenchRunnerConfigImpl) validateServerConfigs() error { + portMap := make(map[int]ServerType) + for _, s := range c.Servers { + st := s.GetServerType() + if st != Dolt && st != MySql && st != Doltgres && st != Postgres { + return fmt.Errorf("unsupported server type: %s", st) + } + + err := s.Validate() + if err != nil { + return err + } + + err = s.SetDefaults() + if err != nil { + return err + } + + //err := ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat) + //if err != nil { + // return err + //} + + //if s.Server == MySql { + // err = CheckProtocol(s.ConnectionProtocol) + // if err != nil { + // return err + // } + //} + // + //if s.Host == "" { + // s.Host = defaultHost + //} + + portMap, err = CheckUpdatePortMap(s, portMap) + if err != nil { + return err + } + + //err = CheckExec(s.ServerExec, "server exec") + //if err != nil { + // return err + //} + // + //if s.Server == Postgres { + // err = CheckExec(s.InitExec, "initdb exec") + // if err != nil { + // return err + // } + //} + // + //if s.Server != Dolt && s.ServerProfile != "" { + // return fmt.Errorf("profiling can only be done against a dolt server") + //} + // + //if s.Server == Dolt && s.ServerProfile != "" { + // if s.ServerProfile != cpuProfile { + // return fmt.Errorf("unsupported server profile: %s", s.ServerProfile) + // } + // if s.ProfilePath == "" { + // cwd, err := os.Getwd() + // if err != nil { + // return err + // } + // s.ProfilePath = cwd + // } + //} + } + + return nil +} + +func (c *sysbenchRunnerConfigImpl) ContainsServerOfType(st ServerType) bool { + for _, s := range c.Servers { + if s.Server == st { + return true + } + } + return false +} + +func ValidateRequiredFields(server, version, format string) error { + if server == "" { + return getMustSupplyError("server") + } + if version == "" { + return getMustSupplyError("version") + } + if format == "" { + return getMustSupplyError("results format") + } + return nil +} + +// setDefaults sets defaults on the sysbenchRunnerConfigImpl +func (c *sysbenchRunnerConfigImpl) setDefaults() error { + if c.RuntimeOS == "" { + c.RuntimeOS = runtime.GOOS + } + if c.RuntimeGoArch == "" { + c.RuntimeGoArch = runtime.GOARCH + } + if len(c.Tests) < 1 { + fmt.Printf("Preparing to benchmark against default tests\n") + if c.ScriptDir != "" { + abs, err := filepath.Abs(c.ScriptDir) + if err != nil { + return err + } + if _, err := os.Stat(abs); os.IsNotExist(err) { + return fmt.Errorf("script dir not found: %s", abs) + } + c.ScriptDir = abs + } + tests, err := getDefaultTests(c) + if err != nil { + return err + } + c.Tests = tests + } + if c.Runs < 1 { + c.Runs = 1 + } + return nil +} + +// CheckUpdatePortMap returns an error if multiple servers have specified the same port +func CheckUpdatePortMap(serverConfig *doltServerConfigImpl, portMap map[int]ServerType) (map[int]ServerType, error) { + if serverConfig.Port == 0 { + serverConfig.Port = defaultPort + } + srv, ok := portMap[serverConfig.Port] + if ok && srv != serverConfig.Server { + return nil, fmt.Errorf("servers have port conflict on port: %d\n", serverConfig.Port) + } + if !ok { + portMap[serverConfig.Port] = serverConfig.Server + } + return portMap, nil +} + +// CheckExec verifies the binary exists +func CheckExec(path, messageIfMissing string) error { + if path == "" { + return getMustSupplyError(messageIfMissing) + } + abs, err := filepath.Abs(path) + if err != nil { + return err + } + if _, err := os.Stat(abs); os.IsNotExist(err) { + return fmt.Errorf("exec not found: %s", abs) + } + return nil +} + +// CheckProtocol ensures the given protocol is supported +func CheckProtocol(protocol string) error { + if protocol == "" { + return getMustSupplyError("connection protocol") + } + if protocol == tcpProtocol || protocol == unixProtocol { + return nil + } + return ErrUnsupportedConnectionProtocol +} + +// GetTests returns a slice of Tests created from the +// defined doltServerConfigImpl.Tests +func GetTests(config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl, testIdFunc func() string) ([]*sysbenchTestImpl, error) { + flattened := make([]*sysbenchTestImpl, 0) + for _, t := range config.Tests { + if len(config.TestOptions) > 0 { + t.Options = append(t.Options, config.TestOptions...) + } + tests, err := t.GetTests(serverConfig, testIdFunc) + if err != nil { + return nil, err + } + flattened = append(flattened, tests...) + } + return flattened, nil +} + +// FromFileConfig returns a validated sysbenchRunnerConfigImpl based on the config file at the configPath +func FromFileConfig(configPath string) (*sysbenchRunnerConfigImpl, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return nil, err + } + + config := NewRunnerConfig() + err = json.Unmarshal(data, config) + if err != nil { + return nil, err + } + + return config, nil +} + +func getMustSupplyError(name string) error { + return fmt.Errorf("Must supply %s", name) +} + +func getDefaultTests(config *sysbenchRunnerConfigImpl) ([]*TestConfig, error) { + defaultTests := make([]*TestConfig, 0) + defaultTests = append(defaultTests, defaultSysbenchTests...) + if config.ScriptDir != "" { + var luaScriptTests []*TestConfig + var err error + if !config.ContainsServerOfType(Doltgres) && !config.ContainsServerOfType(Postgres) { + luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltLuaScripts) + } else { + luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltgresLuaScripts) + } + if err != nil { + return nil, err + } + defaultTests = append(defaultTests, luaScriptTests...) + } + return defaultTests, nil +} + +func getLuaScriptTestsFromDir(dir string, toInclude map[string]string) ([]*TestConfig, error) { + luaScripts := make([]*TestConfig, 0) + abs, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + err = filepath.Walk(abs, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + file := filepath.Base(path) + if _, ok := toInclude[file]; ok { + luaScripts = append(luaScripts, NewConfigTest(path, []string{}, true)) + } + return nil + }) + if err != nil { + return nil, err + } + return luaScripts, nil +} diff --git a/go/performance/utils/sysbench_runner/config_test.go b/go/performance/utils/sysbench_runner/sysbench_config_test.go similarity index 69% rename from go/performance/utils/sysbench_runner/config_test.go rename to go/performance/utils/sysbench_runner/sysbench_config_test.go index 22a0a95037f..8fa83a954d6 100644 --- a/go/performance/utils/sysbench_runner/config_test.go +++ b/go/performance/utils/sysbench_runner/sysbench_config_test.go @@ -23,31 +23,31 @@ import ( var testIdFunc = func() string { return "id" } func TestConfigTestGetTests(t *testing.T) { - empty := &ConfigTest{Name: "test_name"} + empty := &TestConfigImpl{Name: "test_name"} - one := &ConfigTest{Name: "test_one", N: 3} - two := &ConfigTest{Name: "test_two", N: 2} - three := &ConfigTest{Name: "test_three", N: 1} + one := &TestConfigImpl{Name: "test_one", N: 3} + two := &TestConfigImpl{Name: "test_two", N: 2} + three := &TestConfigImpl{Name: "test_three", N: 1} - opts := &ConfigTest{ + opts := &TestConfigImpl{ Name: "test_options", N: 1, Options: []string{"--create_secondary=on", "--auto_inc=off"}, } - serverConfig := &ServerConfig{Server: MySql, Version: "test-version", Host: "localhost", ResultsFormat: CsvFormat} + serverConfig := &doltServerConfigImpl{Server: MySql, Version: "test-version", Host: "localhost", ResultsFormat: CsvFormat} tests := []struct { description string - config *Config - expectedTests []*Test + config *sysbenchRunnerConfigImpl + expectedTests []*sysbenchTestImpl expectedError error }{ { description: "should error if no test name is defined", - config: &Config{ - Servers: []*ServerConfig{serverConfig}, - Tests: []*ConfigTest{ + config: &sysbenchRunnerConfigImpl{ + Servers: []*doltServerConfigImpl{serverConfig}, + Tests: []*TestConfigImpl{ {Name: ""}, }, }, @@ -56,11 +56,11 @@ func TestConfigTestGetTests(t *testing.T) { }, { description: "should create single test if N is < 1", - config: &Config{ - Servers: []*ServerConfig{serverConfig}, - Tests: []*ConfigTest{empty}, + config: &sysbenchRunnerConfigImpl{ + Servers: []*doltServerConfigImpl{serverConfig}, + Tests: []*TestConfigImpl{empty}, }, - expectedTests: []*Test{ + expectedTests: []*sysbenchTestImpl{ { id: testIdFunc(), Name: "test_name", @@ -69,12 +69,12 @@ func TestConfigTestGetTests(t *testing.T) { }, }, { - description: "should return a test for each N defined on the ConfigTest", - config: &Config{ - Servers: []*ServerConfig{serverConfig}, - Tests: []*ConfigTest{one, two, three}, + description: "should return a test for each N defined on the TestConfigImpl", + config: &sysbenchRunnerConfigImpl{ + Servers: []*doltServerConfigImpl{serverConfig}, + Tests: []*TestConfigImpl{one, two, three}, }, - expectedTests: []*Test{ + expectedTests: []*sysbenchTestImpl{ {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, @@ -85,11 +85,11 @@ func TestConfigTestGetTests(t *testing.T) { }, { description: "should apply user options to test params", - config: &Config{ - Servers: []*ServerConfig{serverConfig}, - Tests: []*ConfigTest{opts}, + config: &sysbenchRunnerConfigImpl{ + Servers: []*doltServerConfigImpl{serverConfig}, + Tests: []*TestConfigImpl{opts}, }, - expectedTests: []*Test{ + expectedTests: []*sysbenchTestImpl{ {id: testIdFunc(), Name: "test_options", Params: fromConfigTestParams(opts, serverConfig)}, }, }, diff --git a/go/performance/utils/sysbench_runner/sysbench_tests.go b/go/performance/utils/sysbench_runner/sysbench_tests.go new file mode 100644 index 00000000000..0aadd2fe446 --- /dev/null +++ b/go/performance/utils/sysbench_runner/sysbench_tests.go @@ -0,0 +1,47 @@ +package sysbench_runner + +// sysbenchTestImpl is a single sysbench test +type sysbenchTestImpl struct { + id string + + // Name is the test name + Name string + + // Params are the parameters passed to sysbench + Params []string + + // FromScript indicates if this test is from a lua script + FromScript bool +} + +var _ Test = &sysbenchTestImpl{} + +func NewSysbenchTest(id, name string, params []string, fromScript bool) *sysbenchTestImpl { + return &sysbenchTestImpl{ + id: id, + Name: name, + Params: params, + FromScript: fromScript, + } +} + +// PrepareArgs returns a test's args for sysbench's prepare step +func (t *sysbenchTestImpl) PrepareArgs() []string { + return withCommand(t.Params, "prepare") +} + +// Run returns a test's args for sysbench's run step +func (t *sysbenchTestImpl) RunArgs() []string { + return withCommand(t.Params, "run") +} + +// Cleanup returns a test's args for sysbench's cleanup step +func (t *sysbenchTestImpl) CleanupArgs() []string { + return withCommand(t.Params, "cleanup") +} + +func withCommand(params []string, command string) []string { + c := make([]string, 0) + c = append(c, params...) + return append(c, command) +} diff --git a/go/performance/utils/sysbench_runner/test_config.go b/go/performance/utils/sysbench_runner/test_config.go new file mode 100644 index 00000000000..4cb3e4f819f --- /dev/null +++ b/go/performance/utils/sysbench_runner/test_config.go @@ -0,0 +1,69 @@ +package sysbench_runner + +import "github.com/google/uuid" + +type TestConfig interface { + GetName() string + GetOptions() []string + GetTests(serverConfig ServerConfig) ([]Test, error) + NewId() string +} + +type testConfigImpl struct { + // Name is the test name + Name string + + // N is the number of times a test should run + N int + + // Options are additional sysbench test options a user can supply to run with this test + Options []string + + // FromScript is a boolean indicating that this test is from a lua script + FromScript bool +} + +var _ TestConfig = &testConfigImpl{} + +func NewTestConfig(name string, opts []string, fromScript bool) *testConfigImpl { + options := make([]string, 0) + options = append(options, opts...) + return &testConfigImpl{ + Name: name, + N: 1, + Options: options, + FromScript: fromScript, + } +} + +func (ct *testConfigImpl) NewId() string { + return uuid.New().String() +} + +func (ct *testConfigImpl) GetName() string { + return ct.Name +} + +func (ct *testConfigImpl) GetOptions() []string { + return ct.Options +} + +func (ct *testConfigImpl) GetTests(serverConfig ServerConfig) ([]Test, error) { + if ct.Name == "" { + return nil, ErrTestNameNotDefined + } + if ct.N < 1 { + ct.N = 1 + } + + params := serverConfig.GetTestingArgs(ct) + tests := make([]Test, 0) + + for i := 0; i < ct.N; i++ { + p := make([]string, len(params)) + copy(p, params) + tests = append(tests, NewSysbenchTest(ct.NewId(), ct.Name, p, ct.FromScript)) + } + + return tests, nil +} diff --git a/go/performance/utils/sysbench_runner/tester.go b/go/performance/utils/sysbench_runner/tester.go index 68bf22dac76..0dceaeea87a 100644 --- a/go/performance/utils/sysbench_runner/tester.go +++ b/go/performance/utils/sysbench_runner/tester.go @@ -5,3 +5,9 @@ import "context" type Tester interface { Test(ctx context.Context) (*Result, error) } + +type Test interface { + PrepareArgs() []string + RunArgs() []string + CleanupArgs() []string +} diff --git a/go/performance/utils/sysbench_runner/tpcc.go b/go/performance/utils/sysbench_runner/tpcc.go index edec15b623a..57c8c37c62a 100644 --- a/go/performance/utils/sysbench_runner/tpcc.go +++ b/go/performance/utils/sysbench_runner/tpcc.go @@ -8,7 +8,7 @@ import ( type tpccTestImpl struct { test *TpccTest config *TpccBenchmarkConfig - serverConfig *ServerConfig + serverConfig *doltServerConfigImpl serverParams []string stampFunc func() string idFunc func() string @@ -17,7 +17,7 @@ type tpccTestImpl struct { var _ Tester = &tpccTestImpl{} -func NewTpccTester(config *TpccBenchmarkConfig, serverConfig *ServerConfig, test *TpccTest, serverParams []string, stampFunc func() string) *tpccTestImpl { +func NewTpccTester(config *TpccBenchmarkConfig, serverConfig *doltServerConfigImpl, test *TpccTest, serverParams []string, stampFunc func() string) *tpccTestImpl { return &tpccTestImpl{ config: config, serverParams: serverParams, diff --git a/go/performance/utils/sysbench_runner/tpcc_config.go b/go/performance/utils/sysbench_runner/tpcc_config.go index 2e275a5631c..b9492f0d870 100644 --- a/go/performance/utils/sysbench_runner/tpcc_config.go +++ b/go/performance/utils/sysbench_runner/tpcc_config.go @@ -53,7 +53,7 @@ type TpccBenchmarkConfig struct { ScriptDir string // Servers are the servers to benchmark. - Servers []*ServerConfig + Servers []*doltServerConfigImpl // ScaleFactors represent the scale at which to run each TpccBenchmark at. ScaleFactors []int @@ -64,7 +64,7 @@ type TpccBenchmarkConfig struct { func NewTpccConfig() *TpccBenchmarkConfig { return &TpccBenchmarkConfig{ - Servers: make([]*ServerConfig, 0), + Servers: make([]*doltServerConfigImpl, 0), ScaleFactors: make([]int, 0), } } @@ -196,7 +196,7 @@ func NewTpccTest(name string, params *TpccTestParams) *TpccTest { } // getArgs returns a test's args for all TPCC steps -func (t *TpccTest) getArgs(serverConfig *ServerConfig) []string { +func (t *TpccTest) getArgs(serverConfig *doltServerConfigImpl) []string { params := make([]string, 0) params = append(params, defaultTpccParams...) @@ -222,19 +222,19 @@ func (t *TpccTest) getArgs(serverConfig *ServerConfig) []string { } // TpccPrepare prepares the command executable for the Prepare step. -func (t *TpccTest) TpccPrepare(ctx context.Context, serverConfig *ServerConfig, scriptDir string) *exec.Cmd { +func (t *TpccTest) TpccPrepare(ctx context.Context, serverConfig *doltServerConfigImpl, scriptDir string) *exec.Cmd { cmd := ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "prepare")...) return addParamsToCmd(cmd, scriptDir) } // TpccRun prepares the command executable for the Run step. -func (t *TpccTest) TpccRun(ctx context.Context, serverConfig *ServerConfig, scriptDir string) *exec.Cmd { +func (t *TpccTest) TpccRun(ctx context.Context, serverConfig *doltServerConfigImpl, scriptDir string) *exec.Cmd { cmd := exec.CommandContext(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "run")...) return addParamsToCmd(cmd, scriptDir) } // TpccCleanup prepares the cleanup executable for the Cleanup step. -func (t *TpccTest) TpccCleanup(ctx context.Context, serverConfig *ServerConfig, scriptDir string) *exec.Cmd { +func (t *TpccTest) TpccCleanup(ctx context.Context, serverConfig *doltServerConfigImpl, scriptDir string) *exec.Cmd { cmd := ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "cleanup")...) return addParamsToCmd(cmd, scriptDir) } @@ -246,7 +246,7 @@ func addParamsToCmd(cmd *exec.Cmd, scriptDir string) *exec.Cmd { return cmd } -// FromFileTpccConfig returns a validated Config based on the config file at the configPath +// FromFileTpccConfig returns a validated sysbenchRunnerConfigImpl based on the config file at the configPath func FromFileTpccConfig(configPath string) (*TpccBenchmarkConfig, error) { data, err := os.ReadFile(configPath) if err != nil { From ce120e9fd9157d5adbfcd6d077b23b64c00276e1 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Tue, 13 Feb 2024 11:23:08 -0800 Subject: [PATCH 14/25] /go/performance/utils/sysbench_runner: wip, still grinding on refactor --- .../utils/sysbench_runner/config.go | 13 + .../utils/sysbench_runner/constants.go | 103 +++++ go/performance/utils/sysbench_runner/dolt.go | 36 +- .../sysbench_runner/dolt_server_config.go | 19 + .../utils/sysbench_runner/dolt_tpcc.go | 25 +- .../sysbench_runner/doltgres_server_config.go | 15 + .../sysbench_runner/mysql_server_config.go | 15 + .../sysbench_runner/postgres_server_config.go | 15 + .../utils/sysbench_runner/server.go | 11 +- .../utils/sysbench_runner/server_config.go | 5 + .../utils/sysbench_runner/sysbench.go | 56 ++- .../utils/sysbench_runner/sysbench_config.go | 365 +++++------------- .../utils/sysbench_runner/test_config.go | 5 + .../utils/sysbench_runner/tester.go | 18 +- go/performance/utils/sysbench_runner/tpcc.go | 34 +- .../utils/sysbench_runner/tpcc_config.go | 118 +++--- .../utils/sysbench_runner/tpcc_tests.go | 95 +++++ 17 files changed, 523 insertions(+), 425 deletions(-) create mode 100644 go/performance/utils/sysbench_runner/constants.go create mode 100644 go/performance/utils/sysbench_runner/tpcc_tests.go diff --git a/go/performance/utils/sysbench_runner/config.go b/go/performance/utils/sysbench_runner/config.go index 42d21ec2fe9..8a4161c05e1 100644 --- a/go/performance/utils/sysbench_runner/config.go +++ b/go/performance/utils/sysbench_runner/config.go @@ -3,6 +3,19 @@ package sysbench_runner import "context" type Config interface { + GetRuns() int + GetScriptDir() string + GetNomsBinFormat() string + GetRuntimeOs() string + GetRuntimeGoArch() string + GetTestOptions() []string + GetTestConfigs() []TestConfig + Validate(ctx context.Context) error ContainsServerOfType(server ServerType) bool } + +type TpccConfig interface { + Config + GetScaleFactors() []int +} diff --git a/go/performance/utils/sysbench_runner/constants.go b/go/performance/utils/sysbench_runner/constants.go new file mode 100644 index 00000000000..9209171d1ff --- /dev/null +++ b/go/performance/utils/sysbench_runner/constants.go @@ -0,0 +1,103 @@ +package sysbench_runner + +const ( + Dolt ServerType = "dolt" + Doltgres ServerType = "doltgres" + Postgres ServerType = "postgres" + MySql ServerType = "mysql" + + CsvFormat = "csv" + JsonFormat = "json" + + CsvExt = ".csv" + JsonExt = ".json" + + defaultHost = "127.0.0.1" + defaultDoltPort = 3306 + defaultMysqlPort = defaultDoltPort + defaultDoltgresPort = 5432 + defaultPostgresPort = defaultDoltgresPort + + defaultMysqlSocket = "/var/run/mysqld/mysqld.sock" + + tcpProtocol = "tcp" + unixProtocol = "unix" + + sysbenchUsername = "sysbench" + sysbenchUserLocal = "'sysbench'@'localhost'" + sysbenchPassLocal = "sysbenchpass" + sysbenchDbPsModeFlag = "--db-ps-mode" + sysbenchDbPsModeDisable = "disable" + sysbenchRandTypeFlag = "--rand-type" + sysbenchRandTypeUniform = "uniform" + + doltSqlServerCommand = "sql-server" + + userFlag = "--user" + hostFlag = "--host" + portFlag = "--port" + skipBinLogFlag = "--skip-log-bin" + profileFlag = "--prof" + profilePathFlag = "--prof-path" + cpuProfile = "cpu" + doltgresDataDirFlag = "--data-dir" + MysqlDataDirFlag = "--datadir" + MysqlInitializeInsecureFlag = "--initialize-insecure" + cpuProfileFilename = "cpu.pprof" + + sysbenchOltpReadOnlyTestName = "oltp_read_only" + sysbenchOltpInsertTestName = "oltp_insert" + sysbenchBulkInsertTestName = "bulk_insert" + sysbenchOltpPointSelectTestName = "oltp_point_select" + sysbenchSelectRandomPointsTestName = "select_random_points" + sysbenchSelectRandomRangesTestName = "select_random_ranges" + sysbenchOltpWriteOnlyTestName = "oltp_write_only" + sysbenchOltpReadWriteTestName = "oltp_read_write" + sysbenchOltpUpdateIndexTestName = "oltp_update_index" + sysbenchOltpUpdateNonIndexTestName = "oltp_update_non_index" + + sysbenchCoveringIndexScanLuaTestName = "covering_index_scan.lua" + sysbenchGroupByScanLuaTestName = "groupby_scan.lua" + sysbenchIndexJoinLuaTestName = "index_join.lua" + sysbenchIndexJoinScanLuaTestName = "index_join_scan.lua" + sysbenchIndexScanLuaTestName = "index_scan.lua" + sysbenchOltpDeleteInsertLuaTestName = "oltp_delete_insert.lua" + sysbenchTableScanLuaTestName = "table_scan.lua" + sysbenchTypesDeleteInsertLuaTestName = "types_delete_insert.lua" + sysbenchTypesTableScanLuaTestName = "types_table_scan.lua" + + sysbenchCoveringIndexScanPostgresLuaTestName = "covering_index_scan_postgres.lua" + sysbenchGroupByScanPostgresLuaTestName = "groupby_scan_postgres.lua" + sysbenchIndexJoinPostgresLuaTestName = "index_join_postgres.lua" + sysbenchIndexJoinScanPostgresLuaTestName = "index_join_scan_postgres.lua" + sysbenchIndexScanPostgresLuaTestName = "index_scan_postgres.lua" + sysbenchOltpDeleteInsertPostgresLuaTestName = "oltp_delete_insert_postgres.lua" + sysbenchTableScanPostgresLuaTestName = "table_scan_postgres.lua" + sysbenchTypesDeleteInsertPostgresLuaTestName = "types_delete_insert_postgres.lua" + sysbenchTypesTableScanPostgresLuaTestName = "types_table_scan_postgres.lua" + + doltConfigUsernameKey = "user.name" + doltConfigEmailKey = "user.email" + doltBenchmarkUser = "benchmark" + doltBenchmarkEmail = "benchmark@dolthub.com" + doltConfigCommand = "config" + doltConfigGlobalFlag = "--global" + doltConfigGetFlag = "--get" + doltConfigAddFlag = "--add" + doltCloneCommand = "clone" + doltVersionCommand = "version" + doltInitCommand = "init" + dbName = "test" + bigEmptyRepo = "max-hoffman/big-empty" + nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" + + expectedServerKilledErrorMessage = "signal: killed" + expectedServerTerminatedErrorMessage = "signal: terminated" + + sysbenchCommand = "sysbench" + luaPathEnvVarTemplate = "LUA_PATH=%s" + luaPath = "?.lua" + + tpccDbName = "sbt" + tpccScaleFactorTemplate = "tpcc-scale-factor-%d" +) diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go index 58fcdbecd6b..7721c6af37e 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -10,34 +10,17 @@ import ( "github.com/dolthub/dolt/go/store/types" ) -const ( - doltConfigUsernameKey = "user.name" - doltConfigEmailKey = "user.email" - doltBenchmarkUser = "benchmark" - doltBenchmarkEmail = "benchmark@dolthub.com" - doltConfigCommand = "config" - doltConfigGlobalFlag = "--global" - doltConfigGetFlag = "--get" - doltConfigAddFlag = "--add" - doltCloneCommand = "clone" - doltVersionCommand = "version" - doltInitCommand = "init" - dbName = "test" - bigEmptyRepo = "max-hoffman/big-empty" - nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" -) - var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } type doltBenchmarkerImpl struct { dir string // cwd - config *sysbenchRunnerConfigImpl - serverConfig *doltServerConfigImpl + config Config + serverConfig ServerConfig } var _ Benchmarker = &doltBenchmarkerImpl{} -func NewDoltBenchmarker(dir string, config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl) *doltBenchmarkerImpl { +func NewDoltBenchmarker(dir string, config Config, serverConfig ServerConfig) *doltBenchmarkerImpl { return &doltBenchmarkerImpl{ dir: dir, config: config, @@ -46,20 +29,20 @@ func NewDoltBenchmarker(dir string, config *sysbenchRunnerConfigImpl, serverConf } func (b *doltBenchmarkerImpl) updateGlobalConfig(ctx context.Context) error { - err := CheckSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigUsernameKey, doltBenchmarkUser) + err := CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigUsernameKey, doltBenchmarkUser) if err != nil { return err } - return CheckSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigEmailKey, doltBenchmarkEmail) + return CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigEmailKey, doltBenchmarkEmail) } func (b *doltBenchmarkerImpl) checkInstallation(ctx context.Context) error { - version := ExecCommand(ctx, b.serverConfig.ServerExec, doltVersionCommand) + version := ExecCommand(ctx, b.serverConfig.GetServerExec(), doltVersionCommand) return version.Run() } func (b *doltBenchmarkerImpl) initDoltRepo(ctx context.Context) (string, error) { - return InitDoltRepo(ctx, b.dir, b.serverConfig.ServerExec, b.config.NomsBinFormat, dbName) + return InitDoltRepo(ctx, b.dir, b.serverConfig.GetServerExec(), b.config.GetNomsBinFormat(), dbName) } func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { @@ -90,13 +73,14 @@ func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { return nil, err } - tests, err := GetTests(b.config, b.serverConfig, nil) + tests, err := GetTests(b.config, b.serverConfig) if err != nil { return nil, err } results := make(Results, 0) - for i := 0; i < b.config.Runs; i++ { + runs := b.config.GetRuns() + for i := 0; i < runs; i++ { for _, test := range tests { tester := NewSysbenchTester(b.config, b.serverConfig, test, serverParams, stampFunc) r, err := tester.Test(ctx) diff --git a/go/performance/utils/sysbench_runner/dolt_server_config.go b/go/performance/utils/sysbench_runner/dolt_server_config.go index 148f6712069..3213de2db59 100644 --- a/go/performance/utils/sysbench_runner/dolt_server_config.go +++ b/go/performance/utils/sysbench_runner/dolt_server_config.go @@ -61,6 +61,22 @@ func NewDoltServerConfig(version, serverExec, serverUser, host, resultsFormat, p } } +func (sc *doltServerConfigImpl) GetId() string { + return sc.Id +} + +func (sc *doltServerConfigImpl) GetHost() string { + return sc.Host +} + +func (sc *doltServerConfigImpl) GetPort() int { + return sc.Port +} + +func (sc *doltServerConfigImpl) GetVersion() string { + return sc.Version +} + func (sc *doltServerConfigImpl) GetProfilePath() string { return sc.ProfilePath } @@ -128,6 +144,9 @@ func (sc *doltServerConfigImpl) SetDefaults() error { if sc.Host == "" { sc.Host = defaultHost } + if sc.Port < 1 { + sc.Port = defaultDoltPort + } if sc.ServerProfile != "" { if sc.ProfilePath == "" { cwd, err := os.Getwd() diff --git a/go/performance/utils/sysbench_runner/dolt_tpcc.go b/go/performance/utils/sysbench_runner/dolt_tpcc.go index 9721418abe7..222129bd513 100644 --- a/go/performance/utils/sysbench_runner/dolt_tpcc.go +++ b/go/performance/utils/sysbench_runner/dolt_tpcc.go @@ -7,20 +7,15 @@ import ( "syscall" ) -const ( - tpccDbName = "sbt" - tpccScaleFactorTemplate = "tpcc-scale-factor-%d" -) - type doltTpccBenchmarkerImpl struct { dir string // cwd - config *TpccBenchmarkConfig - serverConfig *doltServerConfigImpl + config TpccConfig + serverConfig ServerConfig } var _ Benchmarker = &doltTpccBenchmarkerImpl{} -func NewDoltTpccBenchmarker(dir string, config *TpccBenchmarkConfig, serverConfig *doltServerConfigImpl) *doltTpccBenchmarkerImpl { +func NewDoltTpccBenchmarker(dir string, config TpccConfig, serverConfig ServerConfig) *doltTpccBenchmarkerImpl { return &doltTpccBenchmarkerImpl{ dir: dir, config: config, @@ -29,20 +24,20 @@ func NewDoltTpccBenchmarker(dir string, config *TpccBenchmarkConfig, serverConfi } func (b *doltTpccBenchmarkerImpl) updateGlobalConfig(ctx context.Context) error { - err := CheckSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigUsernameKey, doltBenchmarkUser) + err := CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigUsernameKey, doltBenchmarkUser) if err != nil { return err } - return CheckSetDoltConfig(ctx, b.serverConfig.ServerExec, doltConfigEmailKey, doltBenchmarkEmail) + return CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigEmailKey, doltBenchmarkEmail) } func (b *doltTpccBenchmarkerImpl) checkInstallation(ctx context.Context) error { - version := ExecCommand(ctx, b.serverConfig.ServerExec, doltVersionCommand) + version := ExecCommand(ctx, b.serverConfig.GetServerExec(), doltVersionCommand) return version.Run() } func (b *doltTpccBenchmarkerImpl) initDoltRepo(ctx context.Context) (string, error) { - return InitDoltRepo(ctx, b.dir, b.serverConfig.ServerExec, b.config.NomsBinFormat, tpccDbName) + return InitDoltRepo(ctx, b.dir, b.serverConfig.GetServerExec(), b.config.GetNomsBinFormat(), tpccDbName) } func (b *doltTpccBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { @@ -95,9 +90,9 @@ func (b *doltTpccBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error } // GetTpccTests creates a set of tests that the server needs to be executed on. -func GetTpccTests(config *TpccBenchmarkConfig) []*TpccTest { - tests := make([]*TpccTest, 0) - for _, sf := range config.ScaleFactors { +func GetTpccTests(config TpccConfig) []Test { + tests := make([]Test, 0) + for _, sf := range config.GetScaleFactors() { params := NewDefaultTpccParams() params.ScaleFactor = sf test := NewTpccTest(fmt.Sprintf(tpccScaleFactorTemplate, sf), params) diff --git a/go/performance/utils/sysbench_runner/doltgres_server_config.go b/go/performance/utils/sysbench_runner/doltgres_server_config.go index e997c35e7ae..2ca6b43c76a 100644 --- a/go/performance/utils/sysbench_runner/doltgres_server_config.go +++ b/go/performance/utils/sysbench_runner/doltgres_server_config.go @@ -98,5 +98,20 @@ func (sc *doltgresServerConfigImpl) SetDefaults() error { if sc.Host == "" { sc.Host = defaultHost } + if sc.Port < 1 { + sc.Port = defaultDoltgresPort + } return nil } + +func (sc *doltgresServerConfigImpl) GetId() string { + return sc.Id +} + +func (sc *doltgresServerConfigImpl) GetHost() string { + return sc.Host +} + +func (sc *doltgresServerConfigImpl) GetPort() int { + return sc.Port +} diff --git a/go/performance/utils/sysbench_runner/mysql_server_config.go b/go/performance/utils/sysbench_runner/mysql_server_config.go index 42e075affe6..7f2a9f91386 100644 --- a/go/performance/utils/sysbench_runner/mysql_server_config.go +++ b/go/performance/utils/sysbench_runner/mysql_server_config.go @@ -118,5 +118,20 @@ func (sc *mysqlServerConfigImpl) SetDefaults() error { if sc.Host == "" { sc.Host = defaultHost } + if sc.Port < 1 { + sc.Port = defaultMysqlPort + } return nil } + +func (sc *mysqlServerConfigImpl) GetId() string { + return sc.Id +} + +func (sc *mysqlServerConfigImpl) GetHost() string { + return sc.Host +} + +func (sc *mysqlServerConfigImpl) GetPort() int { + return sc.Port +} diff --git a/go/performance/utils/sysbench_runner/postgres_server_config.go b/go/performance/utils/sysbench_runner/postgres_server_config.go index 77cbbe153f5..955685bd895 100644 --- a/go/performance/utils/sysbench_runner/postgres_server_config.go +++ b/go/performance/utils/sysbench_runner/postgres_server_config.go @@ -107,5 +107,20 @@ func (sc *postgresServerConfigImpl) SetDefaults() error { if sc.Host == "" { sc.Host = defaultHost } + if sc.Port < 1 { + sc.Port = defaultPostgresPort + } return nil } + +func (sc *postgresServerConfigImpl) GetId() string { + return sc.Id +} + +func (sc *postgresServerConfigImpl) GetHost() string { + return sc.Host +} + +func (sc *postgresServerConfigImpl) GetPort() int { + return sc.Port +} diff --git a/go/performance/utils/sysbench_runner/server.go b/go/performance/utils/sysbench_runner/server.go index 44dcc8a07c4..555b26eaad9 100644 --- a/go/performance/utils/sysbench_runner/server.go +++ b/go/performance/utils/sysbench_runner/server.go @@ -11,11 +11,6 @@ import ( "golang.org/x/sync/errgroup" ) -const ( - expectedServerKilledErrorMessage = "signal: killed" - expectedServerTerminatedErrorMessage = "signal: terminated" -) - var ErrServerClosed = errors.New("server was previously closed") type Server interface { @@ -26,7 +21,7 @@ type Server interface { type doltServerImpl struct { dir string - serverConfig *doltServerConfigImpl + serverConfig ServerConfig serverCtx context.Context serverCtxCancelFunc context.CancelFunc server *exec.Cmd @@ -37,11 +32,11 @@ type doltServerImpl struct { var _ Server = &doltServerImpl{} -func NewServer(ctx context.Context, dir string, serverConfig *doltServerConfigImpl, killSignal os.Signal, serverParams []string) *doltServerImpl { +func NewServer(ctx context.Context, dir string, serverConfig ServerConfig, killSignal os.Signal, serverParams []string) *doltServerImpl { withKeyCtx, cancel := context.WithCancel(ctx) gServer, serverCtx := errgroup.WithContext(withKeyCtx) - server := ExecCommand(serverCtx, serverConfig.ServerExec, serverParams...) + server := ExecCommand(serverCtx, serverConfig.ServerExec(), serverParams...) server.Dir = dir quit := make(chan os.Signal, 1) diff --git a/go/performance/utils/sysbench_runner/server_config.go b/go/performance/utils/sysbench_runner/server_config.go index d4b398f354d..a70bb32631c 100644 --- a/go/performance/utils/sysbench_runner/server_config.go +++ b/go/performance/utils/sysbench_runner/server_config.go @@ -3,10 +3,15 @@ package sysbench_runner type ServerType string type ServerConfig interface { + GetId() string + GetHost() string + GetPort() int + GetVersion() string GetServerExec() string GetServerType() ServerType GetServerArgs() ([]string, error) GetTestingArgs(testConfig TestConfig) []string + Validate() error SetDefaults() error } diff --git a/go/performance/utils/sysbench_runner/sysbench.go b/go/performance/utils/sysbench_runner/sysbench.go index 590146be9e1..63b1f1166e6 100644 --- a/go/performance/utils/sysbench_runner/sysbench.go +++ b/go/performance/utils/sysbench_runner/sysbench.go @@ -11,16 +11,10 @@ import ( "github.com/google/uuid" ) -const ( - sysbenchCommand = "sysbench" - luaPathEnvVarTemplate = "LUA_PATH=%s" - luaPath = "?.lua" -) - type sysbenchTesterImpl struct { - test *sysbenchTestImpl - config *sysbenchRunnerConfigImpl - serverConfig *doltServerConfigImpl + test Test + config Config + serverConfig ServerConfig serverParams []string stampFunc func() string idFunc func() string @@ -29,19 +23,19 @@ type sysbenchTesterImpl struct { var _ Tester = &sysbenchTesterImpl{} -func NewSysbenchTester(config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl, test *sysbenchTestImpl, serverParams []string, stampFunc func() string) *sysbenchTesterImpl { +func NewSysbenchTester(config Config, serverConfig ServerConfig, test Test, serverParams []string, stampFunc func() string) *sysbenchTesterImpl { return &sysbenchTesterImpl{ config: config, serverParams: serverParams, serverConfig: serverConfig, test: test, - suiteId: serverConfig.GetId(), + suiteId: serverConfig.Id(), stampFunc: stampFunc, } } func (t *sysbenchTesterImpl) newResult() (*Result, error) { - serverParams, err := t.serverConfig.GetServerArgs() + serverParams, err := t.serverConfig.ServerArgs() if err != nil { return nil, err } @@ -56,36 +50,36 @@ func (t *sysbenchTesterImpl) newResult() (*Result, error) { } var name string - if t.test.FromScript { - base := filepath.Base(t.test.Name) + if t.test.FromScript() { + base := filepath.Base(t.test.Name()) ext := filepath.Ext(base) name = strings.TrimSuffix(base, ext) } else { - name = t.test.Name + name = t.test.Name() } return &Result{ Id: getId(), SuiteId: t.suiteId, - TestId: t.test.id, - RuntimeOS: t.config.RuntimeOS, - RuntimeGoArch: t.config.RuntimeGoArch, - ServerName: string(t.serverConfig.Server), - ServerVersion: t.serverConfig.Version, + TestId: t.test.Id(), + RuntimeOS: t.config.RuntimeOs(), + RuntimeGoArch: t.config.RuntimeGoArch(), + ServerName: string(t.serverConfig.ServerType()), + ServerVersion: t.serverConfig.Version(), ServerParams: strings.Join(serverParams, " "), TestName: name, - TestParams: strings.Join(t.test.Params, " "), + TestParams: strings.Join(t.test.ParamsToSlice(), " "), }, nil } func (t *sysbenchTesterImpl) outputToResult(output []byte) (*Result, error) { - return OutputToResult(output, t.serverConfig.Server, t.serverConfig.Version, t.test.Name, t.test.id, t.suiteId, t.config.RuntimeOS, t.config.RuntimeGoArch, t.serverParams, t.test.Params, nil, t.test.FromScript) + return OutputToResult(output, t.serverConfig.ServerType(), t.serverConfig.Version(), t.test.Name(), t.test.Id(), t.suiteId, t.config.RuntimeOs(), t.config.RuntimeGoArch(), t.serverParams, t.test.ParamsToSlice(), nil, t.test.FromScript()) } func (t *sysbenchTesterImpl) prepare(ctx context.Context) error { - cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.Prepare()...) - if t.test.FromScript { - lp := filepath.Join(t.config.ScriptDir, luaPath) + cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.PrepareArgs()...) + if t.test.FromScript() { + lp := filepath.Join(t.config.ScriptDir(), luaPath) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) } @@ -98,9 +92,9 @@ func (t *sysbenchTesterImpl) prepare(ctx context.Context) error { } func (t *sysbenchTesterImpl) run(ctx context.Context) (*Result, error) { - cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.Run()...) - if t.test.FromScript { - lp := filepath.Join(t.config.ScriptDir, luaPath) + cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.RunArgs()...) + if t.test.FromScript() { + lp := filepath.Join(t.config.ScriptDir(), luaPath) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) } @@ -126,9 +120,9 @@ func (t *sysbenchTesterImpl) run(ctx context.Context) (*Result, error) { } func (t *sysbenchTesterImpl) cleanup(ctx context.Context) error { - cmd := ExecCommand(ctx, sysbenchCommand, t.test.Cleanup()...) - if t.test.FromScript { - lp := filepath.Join(t.config.ScriptDir, luaPath) + cmd := ExecCommand(ctx, sysbenchCommand, t.test.CleanupArgs()...) + if t.test.FromScript() { + lp := filepath.Join(t.config.ScriptDir(), luaPath) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) } diff --git a/go/performance/utils/sysbench_runner/sysbench_config.go b/go/performance/utils/sysbench_runner/sysbench_config.go index 9785443b2a8..1a8e99d3a3b 100644 --- a/go/performance/utils/sysbench_runner/sysbench_config.go +++ b/go/performance/utils/sysbench_runner/sysbench_config.go @@ -25,43 +25,6 @@ import ( "runtime" ) -const ( - Dolt ServerType = "dolt" - Doltgres ServerType = "doltgres" - Postgres ServerType = "postgres" - MySql ServerType = "mysql" - - CsvFormat = "csv" - JsonFormat = "json" - - CsvExt = ".csv" - JsonExt = ".json" - - defaultHost = "127.0.0.1" - defaultPort = 3306 - - defaultMysqlSocket = "/var/run/mysqld/mysqld.sock" - - tcpProtocol = "tcp" - unixProtocol = "unix" - - sysbenchUsername = "sysbench" - sysbenchUserLocal = "'sysbench'@'localhost'" - sysbenchPassLocal = "sysbenchpass" - - userFlag = "--user" - hostFlag = "--host" - portFlag = "--port" - skipBinLogFlag = "--skip-log-bin" - profileFlag = "--prof" - profilePathFlag = "--prof-path" - cpuProfile = "cpu" - doltgresDataDirFlag = "--data-dir" - MysqlDataDirFlag = "--datadir" - MysqlInitializeInsecureFlag = "--initialize-insecure" - cpuProfileFilename = "cpu.pprof" -) - var ( ErrTestNameNotDefined = errors.New("test name not defined") ErrNoServersDefined = errors.New("servers not defined") @@ -70,150 +33,51 @@ var ( ) var defaultSysbenchParams = []string{ - "--db-ps-mode=disable", - "--rand-type=uniform", + fmt.Sprintf("%s=%s", sysbenchDbPsModeFlag, sysbenchDbPsModeDisable), + fmt.Sprintf("%s=%s", sysbenchRandTypeFlag, sysbenchRandTypeUniform), } -var defaultDoltServerParams = []string{"sql-server"} +var defaultDoltServerParams = []string{doltSqlServerCommand} var defaultSysbenchTests = []TestConfig{ - NewTestConfig("oltp_read_only", []string{}, false), - NewTestConfig("oltp_insert", []string{}, false), - NewTestConfig("bulk_insert", []string{}, false), - NewTestConfig("oltp_point_select", []string{}, false), - NewTestConfig("select_random_points", []string{}, false), - NewTestConfig("select_random_ranges", []string{}, false), - NewTestConfig("oltp_write_only", []string{}, false), - NewTestConfig("oltp_read_write", []string{}, false), - NewTestConfig("oltp_update_index", []string{}, false), - NewTestConfig("oltp_update_non_index", []string{}, false), + NewTestConfig(sysbenchOltpReadOnlyTestName, []string{}, false), + NewTestConfig(sysbenchOltpInsertTestName, []string{}, false), + NewTestConfig(sysbenchBulkInsertTestName, []string{}, false), + NewTestConfig(sysbenchOltpPointSelectTestName, []string{}, false), + NewTestConfig(sysbenchSelectRandomPointsTestName, []string{}, false), + NewTestConfig(sysbenchSelectRandomRangesTestName, []string{}, false), + NewTestConfig(sysbenchOltpWriteOnlyTestName, []string{}, false), + NewTestConfig(sysbenchOltpReadWriteTestName, []string{}, false), + NewTestConfig(sysbenchOltpUpdateIndexTestName, []string{}, false), + NewTestConfig(sysbenchOltpUpdateNonIndexTestName, []string{}, false), } var defaultDoltLuaScripts = map[string]string{ - "covering_index_scan.lua": "covering_index_scan.lua", - "groupby_scan.lua": "groupby_scan.lua", - "index_join.lua": "index_join.lua", - "index_join_scan.lua": "index_join_scan.lua", - "index_scan.lua": "index_scan.lua", - "oltp_delete_insert.lua": "oltp_delete_insert.lua", - "table_scan.lua": "table_scan.lua", - "types_delete_insert.lua": "types_delete_insert.lua", - "types_table_scan.lua": "types_table_scan.lua", + sysbenchCoveringIndexScanLuaTestName: sysbenchCoveringIndexScanLuaTestName, + sysbenchGroupByScanLuaTestName: sysbenchGroupByScanLuaTestName, + sysbenchIndexJoinLuaTestName: sysbenchIndexJoinLuaTestName, + sysbenchIndexJoinScanLuaTestName: sysbenchIndexJoinScanLuaTestName, + sysbenchIndexScanLuaTestName: sysbenchIndexScanLuaTestName, + sysbenchOltpDeleteInsertLuaTestName: sysbenchOltpDeleteInsertLuaTestName, + sysbenchTableScanLuaTestName: sysbenchTableScanLuaTestName, + sysbenchTypesDeleteInsertLuaTestName: sysbenchTypesDeleteInsertLuaTestName, + sysbenchTypesTableScanLuaTestName: sysbenchTypesTableScanLuaTestName, } // todo: check expressions need to be supported in doltgres for these // todo: postgres does not have geometry types also var defaultDoltgresLuaScripts = map[string]string{ - //"covering_index_scan_postgres.lua": "covering_index_scan_postgres.lua", - //"groupby_scan_postgres.lua": "groupby_scan_postgres.lua", - //"index_join_postgres.lua": "index_join_postgres.lua", - //"index_join_scan_postgres.lua": "index_join_scan_postgres.lua", - //"index_scan_postgres.lua": "index_scan_postgres.lua", - //"oltp_delete_insert_postgres.lua": "oltp_delete_insert_postgres.lua", - //"table_scan_postgres.lua": "table_scan_postgres.lua", - //"types_delete_insert_postgres.lua": "types_delete_insert_postgres.lua", - //"types_table_scan_postgres.lua": "types_table_scan_postgres.lua", + //sysbenchCoveringIndexScanPostgresLuaTestName: sysbenchCoveringIndexScanPostgresLuaTestName, + //sysbenchGroupByScanPostgresLuaTestName: sysbenchGroupByScanPostgresLuaTestName, + //sysbenchIndexJoinPostgresLuaTestName: sysbenchIndexJoinPostgresLuaTestName, + //sysbenchIndexJoinScanPostgresLuaTestName: sysbenchIndexJoinScanPostgresLuaTestName, + //sysbenchIndexScanPostgresLuaTestName: sysbenchIndexScanPostgresLuaTestName, + //sysbenchOltpDeleteInsertPostgresLuaTestName: sysbenchOltpDeleteInsertPostgresLuaTestName, + //sysbenchTableScanPostgresLuaTestName: sysbenchTableScanPostgresLuaTestName, + //sysbenchTypesDeleteInsertPostgresLuaTestName: sysbenchTypesDeleteInsertPostgresLuaTestName, + //sysbenchTypesTableScanPostgresLuaTestName: sysbenchTypesTableScanPostgresLuaTestName, } -//// TestConfigImpl provides users a way to define a test for multiple tablesizes -//type TestConfig struct { -// // Name is the test name -// Name string -// -// // N is the number of times a test should run -// N int -// -// // Options are additional sysbench test options a user can supply to run with this test -// Options []string -// -// // FromScript is a boolean indicating that this test is from a lua script -// FromScript bool -//} - -//// NewConfigTest returns a TestConfigImpl containing the supplied args -//func NewConfigTest(name string, opts []string, fromScript bool) *TestConfig { -// options := make([]string, 0) -// options = append(options, opts...) -// return &TestConfig{ -// Name: name, -// N: 1, -// Options: options, -// FromScript: fromScript, -// } -//} - -// GetTests returns a slice of Tests -//func (ct *TestConfig) GetTests(serverConfig *doltServerConfigImpl, testIdFunc func() string) ([]*sysbenchTestImpl, error) { -// if ct.Name == "" { -// return nil, ErrTestNameNotDefined -// } -// if ct.N < 1 { -// ct.N = 1 -// } -// -// params := fromConfigTestParams(ct, serverConfig) -// tests := make([]*sysbenchTestImpl, 0) -// -// var idFunc func() string -// if testIdFunc == nil { -// idFunc = func() string { -// return uuid.New().String() -// } -// } else { -// idFunc = testIdFunc -// } -// -// for i := 0; i < ct.N; i++ { -// p := make([]string, len(params)) -// copy(p, params) -// tests = append(tests, &sysbenchTestImpl{ -// id: idFunc(), -// Name: ct.Name, -// Params: p, -// FromScript: ct.FromScript, -// }) -// } -// return tests, nil -//} - -//// fromConfigTestParams returns params formatted for sysbench: -//// `sysbench [options]... [testname] [command]` -//func fromConfigTestParams(ct *TestConfig, serverConfig *doltServerConfigImpl) []string { -// params := make([]string, 0) -// params = append(params, defaultSysbenchParams...) -// if serverConfig.Server == MySql || serverConfig.Server == Dolt { -// params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) -// params = append(params, "--db-driver=mysql") -// params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host)) -// if serverConfig.Port != 0 { -// params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port)) -// } -// } else if serverConfig.Server == Doltgres || serverConfig.Server == Postgres { -// params = append(params, "--db-driver=pgsql") -// params = append(params, fmt.Sprintf("--pgsql-db=%s", dbName)) -// params = append(params, fmt.Sprintf("--pgsql-host=%s", serverConfig.Host)) -// if serverConfig.Port != 0 { -// params = append(params, fmt.Sprintf("--pgsql-port=%d", serverConfig.Port)) -// } -// } -// -// // handle sysbench user for local mysql server -// if serverConfig.Server == MySql && serverConfig.Host == defaultHost { -// params = append(params, "--mysql-user=sysbench") -// params = append(params, fmt.Sprintf("--mysql-password=%s", sysbenchPassLocal)) -// } else if serverConfig.Server == Dolt { -// params = append(params, "--mysql-user=root") -// } else if serverConfig.Server == Doltgres { -// params = append(params, "--pgsql-user=doltgres") -// } else if serverConfig.Server == Postgres { -// params = append(params, "--pgsql-user=postgres") -// } -// -// params = append(params, ct.Options...) -// params = append(params, ct.Name) -// return params -//} - // sysbenchRunnerConfigImpl is the configuration for a benchmarking run type sysbenchRunnerConfigImpl struct { // Runs is the number of times to run all tests @@ -246,6 +110,34 @@ func NewRunnerConfig() *sysbenchRunnerConfigImpl { } } +func (c *sysbenchRunnerConfigImpl) GetRuns() int { + return c.Runs +} + +func (c *sysbenchRunnerConfigImpl) GetScriptDir() string { + return c.ScriptDir +} + +func (c *sysbenchRunnerConfigImpl) GetNomsBinFormat() string { + return c.NomsBinFormat +} + +func (c *sysbenchRunnerConfigImpl) GetRuntimeOs() string { + return c.RuntimeOS +} + +func (c *sysbenchRunnerConfigImpl) GetRuntimeGoArch() string { + return c.RuntimeGoArch +} + +func (c *sysbenchRunnerConfigImpl) GetTestConfigs() []TestConfig { + return c.Tests +} + +func (c *sysbenchRunnerConfigImpl) GetTestOptions() []string { + return c.TestOptions +} + // Validate checks the config for the required fields and sets defaults // where necessary func (c *sysbenchRunnerConfigImpl) Validate(ctx context.Context) error { @@ -266,7 +158,7 @@ func (c *sysbenchRunnerConfigImpl) Validate(ctx context.Context) error { func (c *sysbenchRunnerConfigImpl) validateServerConfigs() error { portMap := make(map[int]ServerType) for _, s := range c.Servers { - st := s.GetServerType() + st := s.ServerType() if st != Dolt && st != MySql && st != Doltgres && st != Postgres { return fmt.Errorf("unsupported server type: %s", st) } @@ -281,55 +173,10 @@ func (c *sysbenchRunnerConfigImpl) validateServerConfigs() error { return err } - //err := ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat) - //if err != nil { - // return err - //} - - //if s.Server == MySql { - // err = CheckProtocol(s.ConnectionProtocol) - // if err != nil { - // return err - // } - //} - // - //if s.Host == "" { - // s.Host = defaultHost - //} - portMap, err = CheckUpdatePortMap(s, portMap) if err != nil { return err } - - //err = CheckExec(s.ServerExec, "server exec") - //if err != nil { - // return err - //} - // - //if s.Server == Postgres { - // err = CheckExec(s.InitExec, "initdb exec") - // if err != nil { - // return err - // } - //} - // - //if s.Server != Dolt && s.ServerProfile != "" { - // return fmt.Errorf("profiling can only be done against a dolt server") - //} - // - //if s.Server == Dolt && s.ServerProfile != "" { - // if s.ServerProfile != cpuProfile { - // return fmt.Errorf("unsupported server profile: %s", s.ServerProfile) - // } - // if s.ProfilePath == "" { - // cwd, err := os.Getwd() - // if err != nil { - // return err - // } - // s.ProfilePath = cwd - // } - //} } return nil @@ -337,26 +184,13 @@ func (c *sysbenchRunnerConfigImpl) validateServerConfigs() error { func (c *sysbenchRunnerConfigImpl) ContainsServerOfType(st ServerType) bool { for _, s := range c.Servers { - if s.Server == st { + if s.ServerType() == st { return true } } return false } -func ValidateRequiredFields(server, version, format string) error { - if server == "" { - return getMustSupplyError("server") - } - if version == "" { - return getMustSupplyError("version") - } - if format == "" { - return getMustSupplyError("results format") - } - return nil -} - // setDefaults sets defaults on the sysbenchRunnerConfigImpl func (c *sysbenchRunnerConfigImpl) setDefaults() error { if c.RuntimeOS == "" { @@ -377,7 +211,7 @@ func (c *sysbenchRunnerConfigImpl) setDefaults() error { } c.ScriptDir = abs } - tests, err := getDefaultTests(c) + tests, err := c.getDefaultTests() if err != nil { return err } @@ -389,17 +223,35 @@ func (c *sysbenchRunnerConfigImpl) setDefaults() error { return nil } -// CheckUpdatePortMap returns an error if multiple servers have specified the same port -func CheckUpdatePortMap(serverConfig *doltServerConfigImpl, portMap map[int]ServerType) (map[int]ServerType, error) { - if serverConfig.Port == 0 { - serverConfig.Port = defaultPort +func (c *sysbenchRunnerConfigImpl) getDefaultTests() ([]TestConfig, error) { + defaultTests := make([]TestConfig, 0) + defaultTests = append(defaultTests, defaultSysbenchTests...) + if c.ScriptDir != "" { + var luaScriptTests []TestConfig + var err error + if !c.ContainsServerOfType(Doltgres) && !c.ContainsServerOfType(Postgres) { + luaScriptTests, err = getLuaScriptTestsFromDir(c.ScriptDir, defaultDoltLuaScripts) + } else { + luaScriptTests, err = getLuaScriptTestsFromDir(c.ScriptDir, defaultDoltgresLuaScripts) + } + if err != nil { + return nil, err + } + defaultTests = append(defaultTests, luaScriptTests...) } - srv, ok := portMap[serverConfig.Port] - if ok && srv != serverConfig.Server { - return nil, fmt.Errorf("servers have port conflict on port: %d\n", serverConfig.Port) + return defaultTests, nil +} + +// CheckUpdatePortMap returns an error if multiple servers have specified the same port +func CheckUpdatePortMap(serverConfig ServerConfig, portMap map[int]ServerType) (map[int]ServerType, error) { + port := serverConfig.Port() + st := serverConfig.ServerType() + srv, ok := portMap[port] + if ok && srv != st { + return nil, fmt.Errorf("servers have port conflict on port: %d\n", port) } if !ok { - portMap[serverConfig.Port] = serverConfig.Server + portMap[port] = st } return portMap, nil } @@ -430,15 +282,15 @@ func CheckProtocol(protocol string) error { return ErrUnsupportedConnectionProtocol } -// GetTests returns a slice of Tests created from the -// defined doltServerConfigImpl.Tests -func GetTests(config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl, testIdFunc func() string) ([]*sysbenchTestImpl, error) { - flattened := make([]*sysbenchTestImpl, 0) - for _, t := range config.Tests { - if len(config.TestOptions) > 0 { - t.Options = append(t.Options, config.TestOptions...) +// GetTests returns a slice of Tests created from the ServerConfig +func GetTests(config Config, serverConfig ServerConfig) ([]Test, error) { + flattened := make([]Test, 0) + for _, t := range config.GetTestConfigs() { + opts := config.GetTestOptions() + for _, o := range opts { + t.AppendOption(o) } - tests, err := t.GetTests(serverConfig, testIdFunc) + tests, err := t.GetTests(serverConfig) if err != nil { return nil, err } @@ -467,27 +319,8 @@ func getMustSupplyError(name string) error { return fmt.Errorf("Must supply %s", name) } -func getDefaultTests(config *sysbenchRunnerConfigImpl) ([]*TestConfig, error) { - defaultTests := make([]*TestConfig, 0) - defaultTests = append(defaultTests, defaultSysbenchTests...) - if config.ScriptDir != "" { - var luaScriptTests []*TestConfig - var err error - if !config.ContainsServerOfType(Doltgres) && !config.ContainsServerOfType(Postgres) { - luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltLuaScripts) - } else { - luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltgresLuaScripts) - } - if err != nil { - return nil, err - } - defaultTests = append(defaultTests, luaScriptTests...) - } - return defaultTests, nil -} - -func getLuaScriptTestsFromDir(dir string, toInclude map[string]string) ([]*TestConfig, error) { - luaScripts := make([]*TestConfig, 0) +func getLuaScriptTestsFromDir(dir string, toInclude map[string]string) ([]TestConfig, error) { + luaScripts := make([]TestConfig, 0) abs, err := filepath.Abs(dir) if err != nil { return nil, err @@ -499,7 +332,7 @@ func getLuaScriptTestsFromDir(dir string, toInclude map[string]string) ([]*TestC file := filepath.Base(path) if _, ok := toInclude[file]; ok { - luaScripts = append(luaScripts, NewConfigTest(path, []string{}, true)) + luaScripts = append(luaScripts, NewTestConfig(path, []string{}, true)) } return nil }) diff --git a/go/performance/utils/sysbench_runner/test_config.go b/go/performance/utils/sysbench_runner/test_config.go index 4cb3e4f819f..a2e81242989 100644 --- a/go/performance/utils/sysbench_runner/test_config.go +++ b/go/performance/utils/sysbench_runner/test_config.go @@ -5,6 +5,7 @@ import "github.com/google/uuid" type TestConfig interface { GetName() string GetOptions() []string + AppendOption(opt string) GetTests(serverConfig ServerConfig) ([]Test, error) NewId() string } @@ -48,6 +49,10 @@ func (ct *testConfigImpl) GetOptions() []string { return ct.Options } +func (ct *testConfigImpl) AppendOption(opt string) { + ct.Options = append(ct.Options, opt) +} + func (ct *testConfigImpl) GetTests(serverConfig ServerConfig) ([]Test, error) { if ct.Name == "" { return nil, ErrTestNameNotDefined diff --git a/go/performance/utils/sysbench_runner/tester.go b/go/performance/utils/sysbench_runner/tester.go index 0dceaeea87a..7658656bf65 100644 --- a/go/performance/utils/sysbench_runner/tester.go +++ b/go/performance/utils/sysbench_runner/tester.go @@ -7,7 +7,19 @@ type Tester interface { } type Test interface { - PrepareArgs() []string - RunArgs() []string - CleanupArgs() []string + GetId() string + GetName() string + GetParamsToSlice() []string + GetPrepareArgs() []string + GetRunArgs() []string + GetCleanupArgs() []string +} + +type SysbenchTest interface { + Test + GetFromScript() bool +} + +type TestParams interface { + ToSlice() []string } diff --git a/go/performance/utils/sysbench_runner/tpcc.go b/go/performance/utils/sysbench_runner/tpcc.go index 57c8c37c62a..c0f1d5a7e30 100644 --- a/go/performance/utils/sysbench_runner/tpcc.go +++ b/go/performance/utils/sysbench_runner/tpcc.go @@ -5,20 +5,20 @@ import ( "fmt" ) -type tpccTestImpl struct { - test *TpccTest - config *TpccBenchmarkConfig - serverConfig *doltServerConfigImpl +type tpccTesterImpl struct { + test Test + config Config + serverConfig ServerConfig serverParams []string stampFunc func() string idFunc func() string suiteId string } -var _ Tester = &tpccTestImpl{} +var _ Tester = &tpccTesterImpl{} -func NewTpccTester(config *TpccBenchmarkConfig, serverConfig *doltServerConfigImpl, test *TpccTest, serverParams []string, stampFunc func() string) *tpccTestImpl { - return &tpccTestImpl{ +func NewTpccTester(config Config, serverConfig ServerConfig, test Test, serverParams []string, stampFunc func() string) *tpccTesterImpl { + return &tpccTesterImpl{ config: config, serverParams: serverParams, serverConfig: serverConfig, @@ -28,12 +28,12 @@ func NewTpccTester(config *TpccBenchmarkConfig, serverConfig *doltServerConfigIm } } -func (t *tpccTestImpl) outputToResult(output []byte) (*Result, error) { - return OutputToResult(output, t.serverConfig.Server, t.serverConfig.Version, t.test.Name, t.test.Id, t.suiteId, t.config.RuntimeOS, t.config.RuntimeGoArch, t.serverParams, t.test.Params.ToSlice(), nil, false) +func (t *tpccTesterImpl) outputToResult(output []byte) (*Result, error) { + return OutputToResult(output, t.serverConfig.GetServerType(), t.serverConfig.GetVersion(), t.test.GetName(), t.test.GetId(), t.suiteId, t.config.GetRuntimeOs(), t.config.GetRuntimeGoArch(), t.serverParams, t.test.GetParamsToSlice(), nil, false) } -func (t *tpccTestImpl) prepare(ctx context.Context) error { - cmd := t.test.TpccPrepare(ctx, t.serverConfig, t.config.ScriptDir) +func (t *tpccTesterImpl) prepare(ctx context.Context) error { + cmd := t.test.TpccPrepare(ctx, t.serverConfig, t.config.GetScriptDir()) out, err := cmd.Output() if err != nil { fmt.Println(string(out)) @@ -42,8 +42,8 @@ func (t *tpccTestImpl) prepare(ctx context.Context) error { return nil } -func (t *tpccTestImpl) run(ctx context.Context) (*Result, error) { - cmd := t.test.TpccRun(ctx, t.serverConfig, t.config.ScriptDir) +func (t *tpccTesterImpl) run(ctx context.Context) (*Result, error) { + cmd := t.test.TpccRun(ctx, t.serverConfig, t.config.GetScriptDir()) out, err := cmd.Output() if err != nil { fmt.Print(string(out)) @@ -64,18 +64,18 @@ func (t *tpccTestImpl) run(ctx context.Context) (*Result, error) { return rs, nil } -func (t *tpccTestImpl) cleanup(ctx context.Context) error { - cmd := t.test.TpccCleanup(ctx, t.serverConfig, t.config.ScriptDir) +func (t *tpccTesterImpl) cleanup(ctx context.Context) error { + cmd := t.test.TpccCleanup(ctx, t.serverConfig, t.config.GetScriptDir()) return cmd.Run() } -func (t *tpccTestImpl) Test(ctx context.Context) (*Result, error) { +func (t *tpccTesterImpl) Test(ctx context.Context) (*Result, error) { err := t.prepare(ctx) if err != nil { return nil, err } - fmt.Println("Running test", t.test.Name) + fmt.Println("Running test", t.test.GetName()) rs, err := t.run(ctx) if err != nil { diff --git a/go/performance/utils/sysbench_runner/tpcc_config.go b/go/performance/utils/sysbench_runner/tpcc_config.go index b9492f0d870..d7ff299058d 100644 --- a/go/performance/utils/sysbench_runner/tpcc_config.go +++ b/go/performance/utils/sysbench_runner/tpcc_config.go @@ -126,68 +126,68 @@ func (c *TpccBenchmarkConfig) validateServerConfigs() error { return nil } -// TpccTest encapsulates an End to End prepare, run, cleanup test case. -type TpccTest struct { - // Id represents a unique test id - Id string - - // Name represents the name of the test case - Name string - - // Params are associated parameters this test runs with - Params *TpccTestParams -} - -type TpccTestParams struct { - // NumThreads represents the number of threads running queries concurrently. - NumThreads int - - // ScaleFactor represents the number of warehouse to test this at scale. - ScaleFactor int - - // Tables represents the number of tables created per warehouse. - Tables int - - // TrxLevel represents what transaction level to use - TrxLevel string - - // ReportCSV determines whether to report output as a csv. - ReportCSV bool - - // ReportInterval defines how often the tpcc benchmark outputs performance stats. - ReportInterval int - - // Time represents how long - Time int -} - -func (t *TpccTestParams) ToSlice() []string { - params := make([]string, 0) - params = append(params, fmt.Sprintf("numThreads=%d", t.NumThreads)) - params = append(params, fmt.Sprintf("scaleFactor=%d", t.ScaleFactor)) - params = append(params, fmt.Sprintf("tables=%d", t.Tables)) - params = append(params, fmt.Sprintf("trxLevel=%s", t.TrxLevel)) - params = append(params, fmt.Sprintf("reportCsv=%t", t.ReportCSV)) - params = append(params, fmt.Sprintf("reportInterval=%d", t.ReportInterval)) - params = append(params, fmt.Sprintf("time=%d", t.Time)) - return params -} +//// TpccTest encapsulates an End to End prepare, run, cleanup test case. +//type TpccTest struct { +// // Id represents a unique test id +// Id string +// +// // Name represents the name of the test case +// Name string +// +// // Params are associated parameters this test runs with +// Params *TpccTestParams +//} -// NewDefaultTpccParams returns default TpccTestParams. -func NewDefaultTpccParams() *TpccTestParams { - return &TpccTestParams{ - NumThreads: 2, // TODO: When ready, expose as command line argument. - ScaleFactor: 1, - Tables: 1, - TrxLevel: "RR", - ReportCSV: true, - ReportInterval: 1, - Time: 30, - } -} +//type TpccTestParams struct { +// // NumThreads represents the number of threads running queries concurrently. +// NumThreads int +// +// // ScaleFactor represents the number of warehouse to test this at scale. +// ScaleFactor int +// +// // Tables represents the number of tables created per warehouse. +// Tables int +// +// // TrxLevel represents what transaction level to use +// TrxLevel string +// +// // ReportCSV determines whether to report output as a csv. +// ReportCSV bool +// +// // ReportInterval defines how often the tpcc benchmark outputs performance stats. +// ReportInterval int +// +// // Time represents how long +// Time int +//} +// +//func (t *TpccTestParams) ToSlice() []string { +// params := make([]string, 0) +// params = append(params, fmt.Sprintf("numThreads=%d", t.NumThreads)) +// params = append(params, fmt.Sprintf("scaleFactor=%d", t.ScaleFactor)) +// params = append(params, fmt.Sprintf("tables=%d", t.Tables)) +// params = append(params, fmt.Sprintf("trxLevel=%s", t.TrxLevel)) +// params = append(params, fmt.Sprintf("reportCsv=%t", t.ReportCSV)) +// params = append(params, fmt.Sprintf("reportInterval=%d", t.ReportInterval)) +// params = append(params, fmt.Sprintf("time=%d", t.Time)) +// return params +//} + +//// NewDefaultTpccParams returns default TpccTestParams. +//func NewDefaultTpccParams() *TpccTestParams { +// return &TpccTestParams{ +// NumThreads: 2, // TODO: When ready, expose as command line argument. +// ScaleFactor: 1, +// Tables: 1, +// TrxLevel: "RR", +// ReportCSV: true, +// ReportInterval: 1, +// Time: 30, +// } +//} // NewTpccTest instantiates and returns a TPCC test. -func NewTpccTest(name string, params *TpccTestParams) *TpccTest { +func NewTpccTest(name string, params *TpccTestParams) Test { return &TpccTest{ Id: uuid.New().String(), Name: name, diff --git a/go/performance/utils/sysbench_runner/tpcc_tests.go b/go/performance/utils/sysbench_runner/tpcc_tests.go new file mode 100644 index 00000000000..175df69605a --- /dev/null +++ b/go/performance/utils/sysbench_runner/tpcc_tests.go @@ -0,0 +1,95 @@ +package sysbench_runner + +import "fmt" + +type tpccTestParamsImpl struct { + // NumThreads represents the number of threads running queries concurrently. + NumThreads int + + // ScaleFactor represents the number of warehouse to test this at scale. + ScaleFactor int + + // Tables represents the number of tables created per warehouse. + Tables int + + // TrxLevel represents what transaction level to use + TrxLevel string + + // ReportCSV determines whether to report output as a csv. + ReportCSV bool + + // ReportInterval defines how often the tpcc benchmark outputs performance stats. + ReportInterval int + + // Time represents how long + Time int +} + +var _ TestParams = &tpccTestParamsImpl{} + +// NewDefaultTpccParams returns default TpccTestParams. +func NewDefaultTpccParams() *tpccTestParamsImpl { + return &tpccTestParamsImpl{ + NumThreads: 2, // TODO: When ready, expose as command line argument. + ScaleFactor: 1, + Tables: 1, + TrxLevel: "RR", // todo: move to constants + ReportCSV: true, + ReportInterval: 1, + Time: 30, + } +} + +// todo: fix all these to match the real flags +func (t *tpccTestParamsImpl) ToSlice() []string { + params := make([]string, 0) + params = append(params, fmt.Sprintf("numThreads=%d", t.NumThreads)) + params = append(params, fmt.Sprintf("scaleFactor=%d", t.ScaleFactor)) + params = append(params, fmt.Sprintf("tables=%d", t.Tables)) + params = append(params, fmt.Sprintf("trxLevel=%s", t.TrxLevel)) + params = append(params, fmt.Sprintf("reportCsv=%t", t.ReportCSV)) + params = append(params, fmt.Sprintf("reportInterval=%d", t.ReportInterval)) + params = append(params, fmt.Sprintf("time=%d", t.Time)) + return params +} + +// tpccTestImpl encapsulates an End to End prepare, run, cleanup test case. +type tpccTestImpl struct { + // Id represents a unique test id + Id string + + // Name represents the name of the test case + Name string + + // Params are associated parameters this test runs with + Params TestParams +} + +var _ Test = &tpccTestImpl{} + +func (t *tpccTestImpl) GetId() string { + return t.Id +} + +func (t *tpccTestImpl) GetName() string { + return t.Name +} + +func (t *tpccTestImpl) GetParamsToSlice() []string { + return t.Params.ToSlice() +} + +func (t tpccTestImpl) GetPrepareArgs() []string { + //TODO implement me + panic("implement me") +} + +func (t tpccTestImpl) GetRunArgs() []string { + //TODO implement me + panic("implement me") +} + +func (t tpccTestImpl) GetCleanupArgs() []string { + //TODO implement me + panic("implement me") +} From 53bdaeecadf0778436d59b4005a0f844c6f58096 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Tue, 13 Feb 2024 12:26:16 -0800 Subject: [PATCH 15/25] /go/performance/utils/sysbench_runner: wip, still jamming --- .../utils/sysbench_runner/config.go | 9 +- .../utils/sysbench_runner/constants.go | 34 ++- .../utils/sysbench_runner/sysbench_config.go | 66 +++--- .../utils/sysbench_runner/sysbench_tests.go | 6 +- .../utils/sysbench_runner/tester.go | 13 +- go/performance/utils/sysbench_runner/tpcc.go | 27 ++- .../utils/sysbench_runner/tpcc_config.go | 220 +++++------------- .../utils/sysbench_runner/tpcc_tests.go | 163 +++++++++++-- 8 files changed, 308 insertions(+), 230 deletions(-) diff --git a/go/performance/utils/sysbench_runner/config.go b/go/performance/utils/sysbench_runner/config.go index 8a4161c05e1..0af477b12b8 100644 --- a/go/performance/utils/sysbench_runner/config.go +++ b/go/performance/utils/sysbench_runner/config.go @@ -8,13 +8,16 @@ type Config interface { GetNomsBinFormat() string GetRuntimeOs() string GetRuntimeGoArch() string - GetTestOptions() []string - GetTestConfigs() []TestConfig - Validate(ctx context.Context) error ContainsServerOfType(server ServerType) bool } +type SysbenchConfig interface { + Config + GetTestOptions() []string + GetTestConfigs() []TestConfig +} + type TpccConfig interface { Config GetScaleFactors() []int diff --git a/go/performance/utils/sysbench_runner/constants.go b/go/performance/utils/sysbench_runner/constants.go index 9209171d1ff..e214a201b7d 100644 --- a/go/performance/utils/sysbench_runner/constants.go +++ b/go/performance/utils/sysbench_runner/constants.go @@ -94,10 +94,38 @@ const ( expectedServerKilledErrorMessage = "signal: killed" expectedServerTerminatedErrorMessage = "signal: terminated" - sysbenchCommand = "sysbench" - luaPathEnvVarTemplate = "LUA_PATH=%s" - luaPath = "?.lua" + sysbenchCommand = "sysbench" + sysbenchPrepareCommand = "prepare" + sysbenchRunCommand = "run" + sysbenchCleanupCommand = "cleanup" + luaPathEnvVarTemplate = "LUA_PATH=%s" + luaPath = "?.lua" + + defaultMysqlUser = "root" + + // Note this is built for the SysbenchDocker file. If you want to run locally you'll need to override these variables + // for your local MySQL setup. + tpccUserLocal = "'sysbench'@'localhost'" + tpccPassLocal = "sysbenchpass" tpccDbName = "sbt" tpccScaleFactorTemplate = "tpcc-scale-factor-%d" + + tpccDbDriverFlag = "--db-driver" + mysqlDriver = "mysql" + tpccMysqlUsername = "sysbench" + tpccMysqlDbFlag = "--mysql-db" + tpccMysqlHostFlag = "--mysql-host" + tpccMysqlUserFlag = "--mysql-user" + tpccMysqlPasswordFlag = "--mysql-password" + tpccMysqlPortFlag = "--mysql-port" + tpccTimeFlag = "--time" + tpccThreadsFlag = "--threads" + tpccReportIntervalFlag = "--report_interval" + tpccTablesFlag = "--tables" + tpccScaleFlag = "--scale" + tpccTransactionLevelFlag = "--trx_level" + tpccReportCsv = "reportCsv" + tpccTransactionLevelRr = "RR" + tpccLuaFilename = "tpcc.lua" ) diff --git a/go/performance/utils/sysbench_runner/sysbench_config.go b/go/performance/utils/sysbench_runner/sysbench_config.go index 1a8e99d3a3b..4c5fb23ad26 100644 --- a/go/performance/utils/sysbench_runner/sysbench_config.go +++ b/go/performance/utils/sysbench_runner/sysbench_config.go @@ -101,7 +101,7 @@ type sysbenchRunnerConfigImpl struct { NomsBinFormat string } -var _ Config = &sysbenchRunnerConfigImpl{} +var _ SysbenchConfig = &sysbenchRunnerConfigImpl{} // NewRunnerConfig returns a new sysbenchRunnerConfigImpl func NewRunnerConfig() *sysbenchRunnerConfigImpl { @@ -158,7 +158,7 @@ func (c *sysbenchRunnerConfigImpl) Validate(ctx context.Context) error { func (c *sysbenchRunnerConfigImpl) validateServerConfigs() error { portMap := make(map[int]ServerType) for _, s := range c.Servers { - st := s.ServerType() + st := s.GetServerType() if st != Dolt && st != MySql && st != Doltgres && st != Postgres { return fmt.Errorf("unsupported server type: %s", st) } @@ -184,7 +184,7 @@ func (c *sysbenchRunnerConfigImpl) validateServerConfigs() error { func (c *sysbenchRunnerConfigImpl) ContainsServerOfType(st ServerType) bool { for _, s := range c.Servers { - if s.ServerType() == st { + if s.GetServerType() == st { return true } } @@ -223,6 +223,29 @@ func (c *sysbenchRunnerConfigImpl) setDefaults() error { return nil } +func (c *sysbenchRunnerConfigImpl) getLuaScriptTestsFromDir(toInclude map[string]string) ([]TestConfig, error) { + luaScripts := make([]TestConfig, 0) + abs, err := filepath.Abs(c.ScriptDir) + if err != nil { + return nil, err + } + err = filepath.Walk(abs, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + file := filepath.Base(path) + if _, ok := toInclude[file]; ok { + luaScripts = append(luaScripts, NewTestConfig(path, []string{}, true)) + } + return nil + }) + if err != nil { + return nil, err + } + return luaScripts, nil +} + func (c *sysbenchRunnerConfigImpl) getDefaultTests() ([]TestConfig, error) { defaultTests := make([]TestConfig, 0) defaultTests = append(defaultTests, defaultSysbenchTests...) @@ -230,9 +253,9 @@ func (c *sysbenchRunnerConfigImpl) getDefaultTests() ([]TestConfig, error) { var luaScriptTests []TestConfig var err error if !c.ContainsServerOfType(Doltgres) && !c.ContainsServerOfType(Postgres) { - luaScriptTests, err = getLuaScriptTestsFromDir(c.ScriptDir, defaultDoltLuaScripts) + luaScriptTests, err = c.getLuaScriptTestsFromDir(defaultDoltLuaScripts) } else { - luaScriptTests, err = getLuaScriptTestsFromDir(c.ScriptDir, defaultDoltgresLuaScripts) + luaScriptTests, err = c.getLuaScriptTestsFromDir(defaultDoltgresLuaScripts) } if err != nil { return nil, err @@ -244,8 +267,8 @@ func (c *sysbenchRunnerConfigImpl) getDefaultTests() ([]TestConfig, error) { // CheckUpdatePortMap returns an error if multiple servers have specified the same port func CheckUpdatePortMap(serverConfig ServerConfig, portMap map[int]ServerType) (map[int]ServerType, error) { - port := serverConfig.Port() - st := serverConfig.ServerType() + port := serverConfig.GetPort() + st := serverConfig.GetServerType() srv, ok := portMap[port] if ok && srv != st { return nil, fmt.Errorf("servers have port conflict on port: %d\n", port) @@ -282,8 +305,8 @@ func CheckProtocol(protocol string) error { return ErrUnsupportedConnectionProtocol } -// GetTests returns a slice of Tests created from the ServerConfig -func GetTests(config Config, serverConfig ServerConfig) ([]Test, error) { +// GetTests returns a slice of Tests +func GetTests(config SysbenchConfig, serverConfig ServerConfig) ([]Test, error) { flattened := make([]Test, 0) for _, t := range config.GetTestConfigs() { opts := config.GetTestOptions() @@ -300,7 +323,7 @@ func GetTests(config Config, serverConfig ServerConfig) ([]Test, error) { } // FromFileConfig returns a validated sysbenchRunnerConfigImpl based on the config file at the configPath -func FromFileConfig(configPath string) (*sysbenchRunnerConfigImpl, error) { +func FromFileConfig(configPath string) (SysbenchConfig, error) { data, err := os.ReadFile(configPath) if err != nil { return nil, err @@ -318,26 +341,3 @@ func FromFileConfig(configPath string) (*sysbenchRunnerConfigImpl, error) { func getMustSupplyError(name string) error { return fmt.Errorf("Must supply %s", name) } - -func getLuaScriptTestsFromDir(dir string, toInclude map[string]string) ([]TestConfig, error) { - luaScripts := make([]TestConfig, 0) - abs, err := filepath.Abs(dir) - if err != nil { - return nil, err - } - err = filepath.Walk(abs, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - - file := filepath.Base(path) - if _, ok := toInclude[file]; ok { - luaScripts = append(luaScripts, NewTestConfig(path, []string{}, true)) - } - return nil - }) - if err != nil { - return nil, err - } - return luaScripts, nil -} diff --git a/go/performance/utils/sysbench_runner/sysbench_tests.go b/go/performance/utils/sysbench_runner/sysbench_tests.go index 0aadd2fe446..f9b23a4133a 100644 --- a/go/performance/utils/sysbench_runner/sysbench_tests.go +++ b/go/performance/utils/sysbench_runner/sysbench_tests.go @@ -26,17 +26,17 @@ func NewSysbenchTest(id, name string, params []string, fromScript bool) *sysbenc } // PrepareArgs returns a test's args for sysbench's prepare step -func (t *sysbenchTestImpl) PrepareArgs() []string { +func (t *sysbenchTestImpl) GetPrepareArgs(serverConfig ServerConfig) []string { return withCommand(t.Params, "prepare") } // Run returns a test's args for sysbench's run step -func (t *sysbenchTestImpl) RunArgs() []string { +func (t *sysbenchTestImpl) GetRunArgs(serverConfig ServerConfig) []string { return withCommand(t.Params, "run") } // Cleanup returns a test's args for sysbench's cleanup step -func (t *sysbenchTestImpl) CleanupArgs() []string { +func (t *sysbenchTestImpl) GetCleanupArgs(serverConfig ServerConfig) []string { return withCommand(t.Params, "cleanup") } diff --git a/go/performance/utils/sysbench_runner/tester.go b/go/performance/utils/sysbench_runner/tester.go index 7658656bf65..59c1209caa2 100644 --- a/go/performance/utils/sysbench_runner/tester.go +++ b/go/performance/utils/sysbench_runner/tester.go @@ -10,9 +10,9 @@ type Test interface { GetId() string GetName() string GetParamsToSlice() []string - GetPrepareArgs() []string - GetRunArgs() []string - GetCleanupArgs() []string + GetPrepareArgs(serverConfig ServerConfig) []string + GetRunArgs(serverConfig ServerConfig) []string + GetCleanupArgs(serverConfig ServerConfig) []string } type SysbenchTest interface { @@ -21,5 +21,12 @@ type SysbenchTest interface { } type TestParams interface { + GetNumThreads() int + GetScaleFactor() int + GetTables() int + GetTrxLevel() string + GetReportCSV() bool + GetReportInterval() int + GetTime() int ToSlice() []string } diff --git a/go/performance/utils/sysbench_runner/tpcc.go b/go/performance/utils/sysbench_runner/tpcc.go index c0f1d5a7e30..25753efd5cc 100644 --- a/go/performance/utils/sysbench_runner/tpcc.go +++ b/go/performance/utils/sysbench_runner/tpcc.go @@ -3,12 +3,16 @@ package sysbench_runner import ( "context" "fmt" + "os" + "os/exec" + "path/filepath" ) type tpccTesterImpl struct { test Test config Config serverConfig ServerConfig + tpccCommand string serverParams []string stampFunc func() string idFunc func() string @@ -17,8 +21,9 @@ type tpccTesterImpl struct { var _ Tester = &tpccTesterImpl{} -func NewTpccTester(config Config, serverConfig ServerConfig, test Test, serverParams []string, stampFunc func() string) *tpccTesterImpl { +func NewTpccTester(config TpccConfig, serverConfig ServerConfig, test Test, serverParams []string, stampFunc func() string) *tpccTesterImpl { return &tpccTesterImpl{ + tpccCommand: filepath.Join(config.GetScriptDir(), tpccLuaFilename), config: config, serverParams: serverParams, serverConfig: serverConfig, @@ -33,7 +38,9 @@ func (t *tpccTesterImpl) outputToResult(output []byte) (*Result, error) { } func (t *tpccTesterImpl) prepare(ctx context.Context) error { - cmd := t.test.TpccPrepare(ctx, t.serverConfig, t.config.GetScriptDir()) + args := t.test.GetPrepareArgs(t.serverConfig) + cmd := ExecCommand(ctx, t.tpccCommand, args...) + cmd = t.updateCmdEnv(cmd) out, err := cmd.Output() if err != nil { fmt.Println(string(out)) @@ -43,7 +50,10 @@ func (t *tpccTesterImpl) prepare(ctx context.Context) error { } func (t *tpccTesterImpl) run(ctx context.Context) (*Result, error) { - cmd := t.test.TpccRun(ctx, t.serverConfig, t.config.GetScriptDir()) + args := t.test.GetRunArgs(t.serverConfig) + cmd := ExecCommand(ctx, t.tpccCommand, args...) + cmd = t.updateCmdEnv(cmd) + out, err := cmd.Output() if err != nil { fmt.Print(string(out)) @@ -65,7 +75,9 @@ func (t *tpccTesterImpl) run(ctx context.Context) (*Result, error) { } func (t *tpccTesterImpl) cleanup(ctx context.Context) error { - cmd := t.test.TpccCleanup(ctx, t.serverConfig, t.config.GetScriptDir()) + args := t.test.GetCleanupArgs(t.serverConfig) + cmd := ExecCommand(ctx, t.tpccCommand, args...) + cmd = t.updateCmdEnv(cmd) return cmd.Run() } @@ -84,3 +96,10 @@ func (t *tpccTesterImpl) Test(ctx context.Context) (*Result, error) { return rs, t.cleanup(ctx) } + +func (t *tpccTesterImpl) updateCmdEnv(cmd *exec.Cmd) *exec.Cmd { + lp := filepath.Join(t.config.GetScriptDir(), luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) + return cmd +} diff --git a/go/performance/utils/sysbench_runner/tpcc_config.go b/go/performance/utils/sysbench_runner/tpcc_config.go index d7ff299058d..0165754d31b 100644 --- a/go/performance/utils/sysbench_runner/tpcc_config.go +++ b/go/performance/utils/sysbench_runner/tpcc_config.go @@ -19,30 +19,17 @@ import ( "encoding/json" "fmt" "os" - "os/exec" - "path/filepath" "runtime" - - "github.com/google/uuid" -) - -const ( - defaultUser = "root" - - // Note this is built for the SysbenchDocker file. If you want to run locally you'll need to override these variables - // for your local MySQL setup. - tpccUserLocal = "'sysbench'@'localhost'" - tpccPassLocal = "sysbenchpass" ) var defaultTpccParams = []string{ - fmt.Sprintf("--mysql-db=%s", tpccDbName), - "--db-driver=mysql", + fmt.Sprintf("%s=%s", tpccMysqlDbFlag, tpccDbName), + fmt.Sprintf("%s=%s", tpccDbDriverFlag, mysqlDriver), } -// TpccBenchmarkConfig represents a configuration for an execution of the TPCC Benchmark. It executes a series of tests +// tpccConfigImpl represents a configuration for an execution of the TPCC Benchmark. It executes a series of tests // against different ServerConfigurations. -type TpccBenchmarkConfig struct { +type tpccConfigImpl struct { // RuntimeOS is the platform the benchmarks ran on RuntimeOS string @@ -53,7 +40,7 @@ type TpccBenchmarkConfig struct { ScriptDir string // Servers are the servers to benchmark. - Servers []*doltServerConfigImpl + Servers []ServerConfig // ScaleFactors represent the scale at which to run each TpccBenchmark at. ScaleFactors []int @@ -62,63 +49,81 @@ type TpccBenchmarkConfig struct { NomsBinFormat string } -func NewTpccConfig() *TpccBenchmarkConfig { - return &TpccBenchmarkConfig{ - Servers: make([]*doltServerConfigImpl, 0), +var _ TpccConfig = &tpccConfigImpl{} + +func NewTpccConfig() *tpccConfigImpl { + return &tpccConfigImpl{ + Servers: make([]ServerConfig, 0), ScaleFactors: make([]int, 0), } } -func (c *TpccBenchmarkConfig) updateDefaults() error { - if len(c.Servers) < 1 { - return ErrNoServersDefined +func (c *tpccConfigImpl) GetRuns() int { + return 1 +} + +func (c *tpccConfigImpl) GetScriptDir() string { + return c.ScriptDir +} + +func (c *tpccConfigImpl) GetNomsBinFormat() string { + return c.NomsBinFormat +} + +func (c *tpccConfigImpl) GetRuntimeOs() string { + return c.RuntimeOS +} + +func (c *tpccConfigImpl) GetRuntimeGoArch() string { + return c.RuntimeGoArch +} + +func (c *tpccConfigImpl) ContainsServerOfType(st ServerType) bool { + for _, s := range c.Servers { + if s.GetServerType() == st { + return true + } } + return false +} + +func (c *tpccConfigImpl) GetScaleFactors() []int { + return c.ScaleFactors +} +func (c *tpccConfigImpl) setDefaults() { // TODO: Eventually we need to support scale factors all the way to 10 if len(c.ScaleFactors) == 0 { c.ScaleFactors = append(c.ScaleFactors, 1) } - if c.RuntimeOS == "" { c.RuntimeOS = runtime.GOOS } if c.RuntimeGoArch == "" { c.RuntimeGoArch = runtime.GOARCH } - - return c.validateServerConfigs() } // validateServerConfigs ensures the ServerConfigs are valid -func (c *TpccBenchmarkConfig) validateServerConfigs() error { +func (c *tpccConfigImpl) validateServerConfigs() error { portMap := make(map[int]ServerType) for _, s := range c.Servers { - if s.Server != Dolt && s.Server != MySql { - return fmt.Errorf("unsupported server type: %s", s.Server) + st := s.GetServerType() + if st != Dolt && st != MySql && st != Doltgres && st != Postgres { + return fmt.Errorf("unsupported server type: %s", st) } - err := ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat) + err := s.Validate() if err != nil { return err } - if s.Server == MySql { - err = CheckProtocol(s.ConnectionProtocol) - if err != nil { - return err - } - } - - if s.Host == "" { - s.Host = defaultHost - } - - portMap, err = CheckUpdatePortMap(s, portMap) + err = s.SetDefaults() if err != nil { return err } - err = CheckExec(s.ServerExec, "server exec") + portMap, err = CheckUpdatePortMap(s, portMap) if err != nil { return err } @@ -126,128 +131,19 @@ func (c *TpccBenchmarkConfig) validateServerConfigs() error { return nil } -//// TpccTest encapsulates an End to End prepare, run, cleanup test case. -//type TpccTest struct { -// // Id represents a unique test id -// Id string -// -// // Name represents the name of the test case -// Name string -// -// // Params are associated parameters this test runs with -// Params *TpccTestParams -//} - -//type TpccTestParams struct { -// // NumThreads represents the number of threads running queries concurrently. -// NumThreads int -// -// // ScaleFactor represents the number of warehouse to test this at scale. -// ScaleFactor int -// -// // Tables represents the number of tables created per warehouse. -// Tables int -// -// // TrxLevel represents what transaction level to use -// TrxLevel string -// -// // ReportCSV determines whether to report output as a csv. -// ReportCSV bool -// -// // ReportInterval defines how often the tpcc benchmark outputs performance stats. -// ReportInterval int -// -// // Time represents how long -// Time int -//} -// -//func (t *TpccTestParams) ToSlice() []string { -// params := make([]string, 0) -// params = append(params, fmt.Sprintf("numThreads=%d", t.NumThreads)) -// params = append(params, fmt.Sprintf("scaleFactor=%d", t.ScaleFactor)) -// params = append(params, fmt.Sprintf("tables=%d", t.Tables)) -// params = append(params, fmt.Sprintf("trxLevel=%s", t.TrxLevel)) -// params = append(params, fmt.Sprintf("reportCsv=%t", t.ReportCSV)) -// params = append(params, fmt.Sprintf("reportInterval=%d", t.ReportInterval)) -// params = append(params, fmt.Sprintf("time=%d", t.Time)) -// return params -//} - -//// NewDefaultTpccParams returns default TpccTestParams. -//func NewDefaultTpccParams() *TpccTestParams { -// return &TpccTestParams{ -// NumThreads: 2, // TODO: When ready, expose as command line argument. -// ScaleFactor: 1, -// Tables: 1, -// TrxLevel: "RR", -// ReportCSV: true, -// ReportInterval: 1, -// Time: 30, -// } -//} - -// NewTpccTest instantiates and returns a TPCC test. -func NewTpccTest(name string, params *TpccTestParams) Test { - return &TpccTest{ - Id: uuid.New().String(), - Name: name, - Params: params, +func (c *tpccConfigImpl) Validate(ctx context.Context) error { + if len(c.Servers) < 1 { + return ErrNoServersDefined } -} - -// getArgs returns a test's args for all TPCC steps -func (t *TpccTest) getArgs(serverConfig *doltServerConfigImpl) []string { - params := make([]string, 0) - params = append(params, defaultTpccParams...) - - params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host)) - - // handle sysbench user for local mysql server - if serverConfig.Server == MySql && serverConfig.Host == defaultHost { - params = append(params, fmt.Sprintf("--mysql-user=%s", "sysbench")) - params = append(params, fmt.Sprintf("--mysql-password=%s", tpccPassLocal)) - } else { - params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port)) - params = append(params, fmt.Sprintf("--mysql-user=%s", defaultUser)) + if len(c.Servers) > 2 { + return ErrTooManyServersDefined } - - params = append(params, fmt.Sprintf("--time=%d", t.Params.Time)) - params = append(params, fmt.Sprintf("--threads=%d", t.Params.NumThreads)) - params = append(params, fmt.Sprintf("--report_interval=%d", t.Params.ReportInterval)) - params = append(params, fmt.Sprintf("--tables=%d", t.Params.Tables)) - params = append(params, fmt.Sprintf("--scale=%d", t.Params.ScaleFactor)) - params = append(params, fmt.Sprintf("--trx_level=%s", t.Params.TrxLevel)) - - return params -} - -// TpccPrepare prepares the command executable for the Prepare step. -func (t *TpccTest) TpccPrepare(ctx context.Context, serverConfig *doltServerConfigImpl, scriptDir string) *exec.Cmd { - cmd := ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "prepare")...) - return addParamsToCmd(cmd, scriptDir) -} - -// TpccRun prepares the command executable for the Run step. -func (t *TpccTest) TpccRun(ctx context.Context, serverConfig *doltServerConfigImpl, scriptDir string) *exec.Cmd { - cmd := exec.CommandContext(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "run")...) - return addParamsToCmd(cmd, scriptDir) -} - -// TpccCleanup prepares the cleanup executable for the Cleanup step. -func (t *TpccTest) TpccCleanup(ctx context.Context, serverConfig *doltServerConfigImpl, scriptDir string) *exec.Cmd { - cmd := ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "cleanup")...) - return addParamsToCmd(cmd, scriptDir) -} - -func addParamsToCmd(cmd *exec.Cmd, scriptDir string) *exec.Cmd { - lp := filepath.Join(scriptDir, "?.lua") - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) - return cmd + c.setDefaults() + return c.validateServerConfigs() } -// FromFileTpccConfig returns a validated sysbenchRunnerConfigImpl based on the config file at the configPath -func FromFileTpccConfig(configPath string) (*TpccBenchmarkConfig, error) { +// FromFileTpccConfig returns a validated TpccConfig based on the config file at the configPath +func FromFileTpccConfig(configPath string) (TpccConfig, error) { data, err := os.ReadFile(configPath) if err != nil { return nil, err diff --git a/go/performance/utils/sysbench_runner/tpcc_tests.go b/go/performance/utils/sysbench_runner/tpcc_tests.go index 175df69605a..9bdfe1e96b5 100644 --- a/go/performance/utils/sysbench_runner/tpcc_tests.go +++ b/go/performance/utils/sysbench_runner/tpcc_tests.go @@ -1,6 +1,9 @@ package sysbench_runner -import "fmt" +import ( + "fmt" + "github.com/google/uuid" +) type tpccTestParamsImpl struct { // NumThreads represents the number of threads running queries concurrently. @@ -33,23 +36,50 @@ func NewDefaultTpccParams() *tpccTestParamsImpl { NumThreads: 2, // TODO: When ready, expose as command line argument. ScaleFactor: 1, Tables: 1, - TrxLevel: "RR", // todo: move to constants + TrxLevel: tpccTransactionLevelRr, ReportCSV: true, ReportInterval: 1, Time: 30, } } -// todo: fix all these to match the real flags +func (t *tpccTestParamsImpl) GetNumThreads() int { + return t.NumThreads +} + +func (t *tpccTestParamsImpl) GetScaleFactor() int { + return t.ScaleFactor +} + +func (t *tpccTestParamsImpl) GetTables() int { + return t.Tables +} + +func (t *tpccTestParamsImpl) GetTrxLevel() string { + return t.TrxLevel +} + +func (t *tpccTestParamsImpl) GetReportCSV() bool { + return t.ReportCSV +} + +func (t *tpccTestParamsImpl) GetReportInterval() int { + return t.ReportInterval +} + +func (t *tpccTestParamsImpl) GetTime() int { + return t.Time +} + func (t *tpccTestParamsImpl) ToSlice() []string { params := make([]string, 0) - params = append(params, fmt.Sprintf("numThreads=%d", t.NumThreads)) - params = append(params, fmt.Sprintf("scaleFactor=%d", t.ScaleFactor)) - params = append(params, fmt.Sprintf("tables=%d", t.Tables)) - params = append(params, fmt.Sprintf("trxLevel=%s", t.TrxLevel)) - params = append(params, fmt.Sprintf("reportCsv=%t", t.ReportCSV)) - params = append(params, fmt.Sprintf("reportInterval=%d", t.ReportInterval)) - params = append(params, fmt.Sprintf("time=%d", t.Time)) + params = append(params, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.NumThreads)) + params = append(params, fmt.Sprintf("%s=%d", tpccScaleFlag, t.ScaleFactor)) + params = append(params, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Tables)) + params = append(params, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.TrxLevel)) + params = append(params, fmt.Sprintf("%s=%t", tpccReportCsv, t.ReportCSV)) + params = append(params, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.ReportInterval)) + params = append(params, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Time)) return params } @@ -67,6 +97,92 @@ type tpccTestImpl struct { var _ Test = &tpccTestImpl{} +// NewTpccTest instantiates and returns a TPCC test. +func NewTpccTest(name string, params TestParams) *tpccTestImpl { + return &tpccTestImpl{ + Id: uuid.New().String(), + Name: name, + Params: params, + } +} + +func (t *tpccTestImpl) doltArgs(serverConfig ServerConfig) []string { + args := make([]string, 0) + args = append(args, defaultTpccParams...) + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlHostFlag, serverConfig.GetHost())) + args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort())) + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, defaultMysqlUser)) + args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime())) + args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads())) + args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval())) + args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables())) + args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor())) + args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel())) + return args +} + +func (t *tpccTestImpl) mysqlArgs(serverConfig ServerConfig) []string { + args := make([]string, 0) + args = append(args, defaultTpccParams...) + host := serverConfig.GetHost() + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlHostFlag, host)) + if host == defaultHost { + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, tpccMysqlUsername)) + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlPasswordFlag, tpccPassLocal)) + } else { + args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort())) + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, defaultMysqlUser)) + } + args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime())) + args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads())) + args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval())) + args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables())) + args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor())) + args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel())) + return args +} + +func (t *tpccTestImpl) doltgresArgs(serverConfig ServerConfig) []string { + args := make([]string, 0) + args = append(args, defaultTpccParams...) + args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime())) + args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads())) + args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval())) + args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables())) + args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor())) + args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel())) + return args +} + +func (t *tpccTestImpl) postgresArgs(serverConfig ServerConfig) []string { + args := make([]string, 0) + args = append(args, defaultTpccParams...) + args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime())) + args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads())) + args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval())) + args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables())) + args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor())) + args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel())) + return args +} + +// getArgs returns a test's args for all TPCC steps +func (t *tpccTestImpl) getArgs(serverConfig ServerConfig) []string { + st := serverConfig.GetServerType() + switch st { + case Dolt: + return t.doltArgs(serverConfig) + case Doltgres: + return t.doltgresArgs(serverConfig) + case Postgres: + return t.postgresArgs(serverConfig) + case MySql: + return t.mysqlArgs(serverConfig) + default: + panic(fmt.Sprintf("unexpected server type: %s", st)) + } +} + func (t *tpccTestImpl) GetId() string { return t.Id } @@ -79,17 +195,26 @@ func (t *tpccTestImpl) GetParamsToSlice() []string { return t.Params.ToSlice() } -func (t tpccTestImpl) GetPrepareArgs() []string { - //TODO implement me - panic("implement me") +func (t *tpccTestImpl) GetPrepareArgs(serverConfg ServerConfig) []string { + args := make([]string, 0) + serverArgs := t.getArgs(serverConfg) + args = append(args, serverArgs...) + args = append(args, sysbenchPrepareCommand) + return args } -func (t tpccTestImpl) GetRunArgs() []string { - //TODO implement me - panic("implement me") +func (t *tpccTestImpl) GetRunArgs(serverConfg ServerConfig) []string { + args := make([]string, 0) + serverArgs := t.getArgs(serverConfg) + args = append(args, serverArgs...) + args = append(args, sysbenchRunCommand) + return args } -func (t tpccTestImpl) GetCleanupArgs() []string { - //TODO implement me - panic("implement me") +func (t *tpccTestImpl) GetCleanupArgs(serverConfg ServerConfig) []string { + args := make([]string, 0) + serverArgs := t.getArgs(serverConfg) + args = append(args, serverArgs...) + args = append(args, sysbenchCleanupCommand) + return args } From 6b22c8da8660453a2f67235c7d995339eab4f9a8 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Tue, 13 Feb 2024 12:40:03 -0800 Subject: [PATCH 16/25] /go/performance/utils/sysbench_runner: wip, still grinding --- .../utils/sysbench_runner/constants.go | 12 +++++- go/performance/utils/sysbench_runner/dolt.go | 4 +- .../sysbench_runner/dolt_server_config.go | 2 +- .../utils/sysbench_runner/doltgres.go | 17 +++++---- .../sysbench_runner/doltgres_server_config.go | 6 ++- go/performance/utils/sysbench_runner/mysql.go | 28 ++++---------- .../sysbench_runner/mysql_server_config.go | 6 ++- .../utils/sysbench_runner/server_config.go | 7 +++- .../utils/sysbench_runner/sysbench_tests.go | 37 +++++++++++++++---- .../utils/sysbench_runner/tester.go | 6 ++- 10 files changed, 82 insertions(+), 43 deletions(-) diff --git a/go/performance/utils/sysbench_runner/constants.go b/go/performance/utils/sysbench_runner/constants.go index e214a201b7d..897f6fe4586 100644 --- a/go/performance/utils/sysbench_runner/constants.go +++ b/go/performance/utils/sysbench_runner/constants.go @@ -112,7 +112,6 @@ const ( tpccScaleFactorTemplate = "tpcc-scale-factor-%d" tpccDbDriverFlag = "--db-driver" - mysqlDriver = "mysql" tpccMysqlUsername = "sysbench" tpccMysqlDbFlag = "--mysql-db" tpccMysqlHostFlag = "--mysql-host" @@ -128,4 +127,15 @@ const ( tpccReportCsv = "reportCsv" tpccTransactionLevelRr = "RR" tpccLuaFilename = "tpcc.lua" + + mysqlDriverName = "mysql" + mysqlRootTCPDsnTemplate = "root@tcp(%s:%d)/" + mysqlRootUnixDsnTemplate = "root@unix(%s)/" + mysqlDropDatabaseSqlTemplate = "DROP DATABASE IF EXISTS %s;" + mysqlCreateDatabaseSqlTemplate = "CREATE DATABASE %s;" + mysqlDropUserSqlTemplate = "DROP USER IF EXISTS %s;" + mysqlCreateUserSqlTemplate = "CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s';" + mysqlGrantPermissionsSqlTemplate = "GRANT ALL ON %s.* to %s;" + mysqlSetGlobalLocalInfileSql = "SET GLOBAL local_infile = 'ON';" + mysqlSetGlobalSqlModeSql = "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" ) diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go index 7721c6af37e..5b03293d3a8 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -14,13 +14,13 @@ var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } type doltBenchmarkerImpl struct { dir string // cwd - config Config + config SysbenchConfig serverConfig ServerConfig } var _ Benchmarker = &doltBenchmarkerImpl{} -func NewDoltBenchmarker(dir string, config Config, serverConfig ServerConfig) *doltBenchmarkerImpl { +func NewDoltBenchmarker(dir string, config SysbenchConfig, serverConfig ServerConfig) *doltBenchmarkerImpl { return &doltBenchmarkerImpl{ dir: dir, config: config, diff --git a/go/performance/utils/sysbench_runner/dolt_server_config.go b/go/performance/utils/sysbench_runner/dolt_server_config.go index 3213de2db59..db3f8392bdc 100644 --- a/go/performance/utils/sysbench_runner/dolt_server_config.go +++ b/go/performance/utils/sysbench_runner/dolt_server_config.go @@ -110,7 +110,7 @@ func (sc *doltServerConfigImpl) GetServerArgs() ([]string, error) { func (sc *doltServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { params := make([]string, 0) params = append(params, defaultSysbenchParams...) - params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) + params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) // todo: replace these with consts params = append(params, "--db-driver=mysql") params = append(params, fmt.Sprintf("--mysql-host=%s", sc.Host)) params = append(params, "--mysql-user=root") diff --git a/go/performance/utils/sysbench_runner/doltgres.go b/go/performance/utils/sysbench_runner/doltgres.go index 7e2d250c5cd..daeb7830772 100644 --- a/go/performance/utils/sysbench_runner/doltgres.go +++ b/go/performance/utils/sysbench_runner/doltgres.go @@ -19,13 +19,13 @@ const ( type doltgresBenchmarkerImpl struct { dir string // cwd - config *sysbenchRunnerConfigImpl - serverConfig *doltServerConfigImpl + config SysbenchConfig + serverConfig ServerConfig } var _ Benchmarker = &doltgresBenchmarkerImpl{} -func NewDoltgresBenchmarker(dir string, config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl) *doltgresBenchmarkerImpl { +func NewDoltgresBenchmarker(dir string, config SysbenchConfig, serverConfig ServerConfig) *doltgresBenchmarkerImpl { return &doltgresBenchmarkerImpl{ dir: dir, config: config, @@ -34,7 +34,7 @@ func NewDoltgresBenchmarker(dir string, config *sysbenchRunnerConfigImpl, server } func (b *doltgresBenchmarkerImpl) checkInstallation(ctx context.Context) error { - version := ExecCommand(ctx, b.serverConfig.ServerExec, doltVersionCommand) + version := ExecCommand(ctx, b.serverConfig.GetServerExec(), doltVersionCommand) return version.Run() } @@ -58,7 +58,7 @@ func (b *doltgresBenchmarkerImpl) cleanupServerDir(dir string) error { } func (b *doltgresBenchmarkerImpl) createTestingDb(ctx context.Context) error { - psqlconn := fmt.Sprintf(psqlDsnTemplate, b.serverConfig.Host, b.serverConfig.Port, doltgresUser, "", dbName) + psqlconn := fmt.Sprintf(psqlDsnTemplate, b.serverConfig.GetHost(), b.serverConfig.GetPort(), doltgresUser, "", dbName) // open database db, err := sql.Open(postgresDriver, psqlconn) @@ -116,14 +116,15 @@ func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Result return } - var tests []*sysbenchTestImpl - tests, err = GetTests(b.config, b.serverConfig, nil) + var tests []Test + tests, err = GetTests(b.config, b.serverConfig) if err != nil { return } results = make(Results, 0) - for i := 0; i < b.config.Runs; i++ { + runs := b.config.GetRuns() + for i := 0; i < runs; i++ { for _, test := range tests { tester := NewSysbenchTester(b.config, b.serverConfig, test, serverParams, stampFunc) var r *Result diff --git a/go/performance/utils/sysbench_runner/doltgres_server_config.go b/go/performance/utils/sysbench_runner/doltgres_server_config.go index 2ca6b43c76a..505ebd87b53 100644 --- a/go/performance/utils/sysbench_runner/doltgres_server_config.go +++ b/go/performance/utils/sysbench_runner/doltgres_server_config.go @@ -69,7 +69,7 @@ func (sc *doltgresServerConfigImpl) GetServerArgs() ([]string, error) { func (sc *doltgresServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { params := make([]string, 0) params = append(params, defaultSysbenchParams...) - params = append(params, "--db-driver=pgsql") + params = append(params, "--db-driver=pgsql") // todo: replace with consts params = append(params, fmt.Sprintf("--pgsql-db=%s", dbName)) params = append(params, fmt.Sprintf("--pgsql-host=%s", sc.Host)) params = append(params, "--pgsql-user=doltgres") @@ -115,3 +115,7 @@ func (sc *doltgresServerConfigImpl) GetHost() string { func (sc *doltgresServerConfigImpl) GetPort() int { return sc.Port } + +func (sc *doltgresServerConfigImpl) GetVersion() string { + return sc.Version +} diff --git a/go/performance/utils/sysbench_runner/mysql.go b/go/performance/utils/sysbench_runner/mysql.go index b962feefe64..0907857a75e 100644 --- a/go/performance/utils/sysbench_runner/mysql.go +++ b/go/performance/utils/sysbench_runner/mysql.go @@ -10,28 +10,15 @@ import ( _ "github.com/go-sql-driver/mysql" ) -const ( - mysqlDriverName = "mysql" - mysqlRootTCPDsnTemplate = "root@tcp(%s:%d)/" - mysqlRootUnixDsnTemplate = "root@unix(%s)/" - mysqlDropDatabaseSqlTemplate = "DROP DATABASE IF EXISTS %s;" - mysqlCreateDatabaseSqlTemplate = "CREATE DATABASE %s;" - mysqlDropUserSqlTemplate = "DROP USER IF EXISTS %s;" - mysqlCreateUserSqlTemplate = "CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s';" - mysqlGrantPermissionsSqlTemplate = "GRANT ALL ON %s.* to %s;" - mysqlSetGlobalLocalInfileSql = "SET GLOBAL local_infile = 'ON';" - mysqlSetGlobalSqlModeSql = "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" -) - type mysqlBenchmarkerImpl struct { dir string // cwd - config *sysbenchRunnerConfigImpl - serverConfig *doltServerConfigImpl + config SysbenchConfig + serverConfig ProtocolServerConfig } var _ Benchmarker = &mysqlBenchmarkerImpl{} -func NewMysqlBenchmarker(dir string, config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl) *mysqlBenchmarkerImpl { +func NewMysqlBenchmarker(dir string, config SysbenchConfig, serverConfig ProtocolServerConfig) *mysqlBenchmarkerImpl { return &mysqlBenchmarkerImpl{ dir: dir, config: config, @@ -40,7 +27,7 @@ func NewMysqlBenchmarker(dir string, config *sysbenchRunnerConfigImpl, serverCon } func (b *mysqlBenchmarkerImpl) getDsn() (string, error) { - return GetMysqlDsn(b.serverConfig.Host, b.serverConfig.Socket, b.serverConfig.ConnectionProtocol, b.serverConfig.Port) + return GetMysqlDsn(b.serverConfig.GetHost(), b.serverConfig.GetSocket(), b.serverConfig.GetConnectionProtocol(), b.serverConfig.GetPort()) } func (b *mysqlBenchmarkerImpl) createTestingDb(ctx context.Context) error { @@ -52,7 +39,7 @@ func (b *mysqlBenchmarkerImpl) createTestingDb(ctx context.Context) error { } func (b *mysqlBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { - serverDir, err := InitMysqlDataDir(ctx, b.serverConfig.ServerExec, dbName) + serverDir, err := InitMysqlDataDir(ctx, b.serverConfig.GetServerExec(), dbName) if err != nil { return nil, err } @@ -74,13 +61,14 @@ func (b *mysqlBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { return nil, err } - tests, err := GetTests(b.config, b.serverConfig, nil) + tests, err := GetTests(b.config, b.serverConfig) if err != nil { return nil, err } results := make(Results, 0) - for i := 0; i < b.config.Runs; i++ { + runs := b.config.GetRuns() + for i := 0; i < runs; i++ { for _, test := range tests { tester := NewSysbenchTester(b.config, b.serverConfig, test, serverParams, stampFunc) r, err := tester.Test(ctx) diff --git a/go/performance/utils/sysbench_runner/mysql_server_config.go b/go/performance/utils/sysbench_runner/mysql_server_config.go index 7f2a9f91386..10f7f675e99 100644 --- a/go/performance/utils/sysbench_runner/mysql_server_config.go +++ b/go/performance/utils/sysbench_runner/mysql_server_config.go @@ -84,7 +84,7 @@ func (sc *mysqlServerConfigImpl) GetServerArgs() ([]string, error) { func (sc *mysqlServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { params := make([]string, 0) params = append(params, defaultSysbenchParams...) - params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) + params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) // todo: replace with consts params = append(params, "--db-driver=mysql") params = append(params, fmt.Sprintf("--mysql-host=%s", sc.Host)) if sc.Port != 0 { @@ -135,3 +135,7 @@ func (sc *mysqlServerConfigImpl) GetHost() string { func (sc *mysqlServerConfigImpl) GetPort() int { return sc.Port } + +func (sc *mysqlServerConfigImpl) GetVersion() string { + return sc.Version +} diff --git a/go/performance/utils/sysbench_runner/server_config.go b/go/performance/utils/sysbench_runner/server_config.go index a70bb32631c..7a5d3c29633 100644 --- a/go/performance/utils/sysbench_runner/server_config.go +++ b/go/performance/utils/sysbench_runner/server_config.go @@ -11,7 +11,6 @@ type ServerConfig interface { GetServerType() ServerType GetServerArgs() ([]string, error) GetTestingArgs(testConfig TestConfig) []string - Validate() error SetDefaults() error } @@ -21,6 +20,12 @@ type InitServerConfig interface { GetInitDbExec() string } +type ProtocolServerConfig interface { + ServerConfig + GetConnectionProtocol() string + GetSocket() string +} + type ProfilingServerConfig interface { ServerConfig GetServerProfile() ServerProfile diff --git a/go/performance/utils/sysbench_runner/sysbench_tests.go b/go/performance/utils/sysbench_runner/sysbench_tests.go index f9b23a4133a..39de6509156 100644 --- a/go/performance/utils/sysbench_runner/sysbench_tests.go +++ b/go/performance/utils/sysbench_runner/sysbench_tests.go @@ -1,5 +1,16 @@ package sysbench_runner +type sysbenchTestParamsImpl []string + +func (s *sysbenchTestParamsImpl) ToSlice() []string { + if s != nil { + return *s + } + return []string{} +} + +var _ TestParams = &sysbenchTestParamsImpl{} + // sysbenchTestImpl is a single sysbench test type sysbenchTestImpl struct { id string @@ -8,7 +19,7 @@ type sysbenchTestImpl struct { Name string // Params are the parameters passed to sysbench - Params []string + Params TestParams // FromScript indicates if this test is from a lua script FromScript bool @@ -16,7 +27,7 @@ type sysbenchTestImpl struct { var _ Test = &sysbenchTestImpl{} -func NewSysbenchTest(id, name string, params []string, fromScript bool) *sysbenchTestImpl { +func NewSysbenchTest(id, name string, params TestParams, fromScript bool) *sysbenchTestImpl { return &sysbenchTestImpl{ id: id, Name: name, @@ -25,23 +36,35 @@ func NewSysbenchTest(id, name string, params []string, fromScript bool) *sysbenc } } +func (t *sysbenchTestImpl) GetId() string { + return t.id +} + +func (t *sysbenchTestImpl) GetName() string { + return t.Name +} + +func (t *sysbenchTestImpl) GetParamsToSlice() []string { + return t.Params.ToSlice() +} + // PrepareArgs returns a test's args for sysbench's prepare step func (t *sysbenchTestImpl) GetPrepareArgs(serverConfig ServerConfig) []string { - return withCommand(t.Params, "prepare") + return withCommand(t.Params, sysbenchPrepareCommand) } // Run returns a test's args for sysbench's run step func (t *sysbenchTestImpl) GetRunArgs(serverConfig ServerConfig) []string { - return withCommand(t.Params, "run") + return withCommand(t.Params, sysbenchRunCommand) } // Cleanup returns a test's args for sysbench's cleanup step func (t *sysbenchTestImpl) GetCleanupArgs(serverConfig ServerConfig) []string { - return withCommand(t.Params, "cleanup") + return withCommand(t.Params, sysbenchCleanupCommand) } -func withCommand(params []string, command string) []string { +func withCommand(params TestParams, command string) []string { c := make([]string, 0) - c = append(c, params...) + c = append(c, params.ToSlice()...) return append(c, command) } diff --git a/go/performance/utils/sysbench_runner/tester.go b/go/performance/utils/sysbench_runner/tester.go index 59c1209caa2..bb104a9d702 100644 --- a/go/performance/utils/sysbench_runner/tester.go +++ b/go/performance/utils/sysbench_runner/tester.go @@ -21,6 +21,11 @@ type SysbenchTest interface { } type TestParams interface { + ToSlice() []string +} + +type TpccTestParams interface { + TestParams GetNumThreads() int GetScaleFactor() int GetTables() int @@ -28,5 +33,4 @@ type TestParams interface { GetReportCSV() bool GetReportInterval() int GetTime() int - ToSlice() []string } From 05bf46fb7468fb108810b5810408c11ed19b9d1e Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Tue, 13 Feb 2024 14:39:12 -0800 Subject: [PATCH 17/25] /go/performance/utils/sysbench_runner: wip, need to fix tests, then test it all --- .../utils/sysbench_runner/config.go | 1 + .../utils/sysbench_runner/constants.go | 68 +++++++++++++-- go/performance/utils/sysbench_runner/csv.go | 2 +- go/performance/utils/sysbench_runner/dolt.go | 9 +- .../sysbench_runner/dolt_server_config.go | 28 +++--- .../utils/sysbench_runner/doltgres.go | 14 ++- .../sysbench_runner/doltgres_server_config.go | 24 +++--- go/performance/utils/sysbench_runner/mysql.go | 6 +- .../sysbench_runner/mysql_server_config.go | 58 +++++++------ .../utils/sysbench_runner/mysql_tpcc.go | 10 +-- .../utils/sysbench_runner/postgres.go | 36 +++----- .../sysbench_runner/postgres_server_config.go | 52 ++++++----- .../utils/sysbench_runner/profile.go | 27 +++--- .../utils/sysbench_runner/results.go | 26 +----- .../utils/sysbench_runner/results_test.go | 59 ++++++++++++- go/performance/utils/sysbench_runner/run.go | 86 ++++++++++++------- .../utils/sysbench_runner/run_tpcc.go | 21 +++-- .../utils/sysbench_runner/server.go | 2 +- .../utils/sysbench_runner/server_config.go | 3 +- .../utils/sysbench_runner/sysbench.go | 48 +++++------ .../utils/sysbench_runner/sysbench_config.go | 4 + .../utils/sysbench_runner/sysbench_tests.go | 19 ++-- .../utils/sysbench_runner/test_config.go | 8 +- .../utils/sysbench_runner/tester.go | 5 ++ .../utils/sysbench_runner/tpcc_config.go | 6 +- .../utils/sysbench_runner/tpcc_tests.go | 4 +- 26 files changed, 390 insertions(+), 236 deletions(-) diff --git a/go/performance/utils/sysbench_runner/config.go b/go/performance/utils/sysbench_runner/config.go index 0af477b12b8..50ee8a11808 100644 --- a/go/performance/utils/sysbench_runner/config.go +++ b/go/performance/utils/sysbench_runner/config.go @@ -8,6 +8,7 @@ type Config interface { GetNomsBinFormat() string GetRuntimeOs() string GetRuntimeGoArch() string + GetServerConfigs() []ServerConfig Validate(ctx context.Context) error ContainsServerOfType(server ServerType) bool } diff --git a/go/performance/utils/sysbench_runner/constants.go b/go/performance/utils/sysbench_runner/constants.go index 897f6fe4586..63440d37e26 100644 --- a/go/performance/utils/sysbench_runner/constants.go +++ b/go/performance/utils/sysbench_runner/constants.go @@ -1,5 +1,7 @@ package sysbench_runner +import "time" + const ( Dolt ServerType = "dolt" Doltgres ServerType = "doltgres" @@ -12,6 +14,8 @@ const ( CsvExt = ".csv" JsonExt = ".json" + CpuServerProfile ServerProfile = "cpu" + defaultHost = "127.0.0.1" defaultDoltPort = 3306 defaultMysqlPort = defaultDoltPort @@ -23,13 +27,24 @@ const ( tcpProtocol = "tcp" unixProtocol = "unix" - sysbenchUsername = "sysbench" - sysbenchUserLocal = "'sysbench'@'localhost'" - sysbenchPassLocal = "sysbenchpass" - sysbenchDbPsModeFlag = "--db-ps-mode" - sysbenchDbPsModeDisable = "disable" - sysbenchRandTypeFlag = "--rand-type" - sysbenchRandTypeUniform = "uniform" + sysbenchUsername = "sysbench" + sysbenchUserLocal = "'sysbench'@'localhost'" + sysbenchPassLocal = "sysbenchpass" + sysbenchDbPsModeFlag = "--db-ps-mode" + sysbenchDbPsModeDisable = "disable" + sysbenchRandTypeFlag = "--rand-type" + sysbenchRandTypeUniform = "uniform" + sysbenchMysqlDbFlag = "--mysql-db" + sysbenchDbDriverFlag = "--db-driver" + sysbenchMysqlHostFlag = "--mysql-host" + sysbenchMysqlPortFlag = "--mysql-port" + sysbenchMysqlUserFlag = "--mysql-user" + sysbenchMysqlPasswordFlag = "--mysql-password" + sysbenchPostgresDbDriver = "pgsql" + sysbenchPostgresDbFlag = "--pgsql-db" + sysbenchPostgresHostFlag = "--pgsql-host" + sysbenchPostgresPortFlag = "--pgsql-port" + sysbenchPostgresUserFlag = "--pgsql-user" doltSqlServerCommand = "sql-server" @@ -91,10 +106,17 @@ const ( bigEmptyRepo = "max-hoffman/big-empty" nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" + postgresDriver = "postgres" + doltgresUser = "doltgres" + doltDataDir = ".dolt" + createDatabaseTemplate = "create database %s;" + psqlDsnTemplate = "host=%s port=%d user=%s password=%s dbname=%s sslmode=disable" + expectedServerKilledErrorMessage = "signal: killed" expectedServerTerminatedErrorMessage = "signal: terminated" sysbenchCommand = "sysbench" + sysbenchVersionFlag = "--version" sysbenchPrepareCommand = "prepare" sysbenchRunCommand = "run" sysbenchCleanupCommand = "cleanup" @@ -138,4 +160,36 @@ const ( mysqlGrantPermissionsSqlTemplate = "GRANT ALL ON %s.* to %s;" mysqlSetGlobalLocalInfileSql = "SET GLOBAL local_infile = 'ON';" mysqlSetGlobalSqlModeSql = "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" + + postgresInitDbDataDirFlag = "--pgdata" + postgresUsernameFlag = "--username" + postgresUsername = "postgres" + postgresDataDirFlag = "-D" + postgresDropDatabaseSqlTemplate = "DROP DATABASE IF EXISTS %s;" + postgresDropUserSqlTemplate = "DROP USER IF EXISTS %s;" + postgresCreateUserSqlTemplate = "CREATE USER %s WITH PASSWORD '%s';" + postgresCreateDatabaseSqlTemplate = "CREATE DATABASE %s WITH OWNER %s;" + postgresLcAllEnvVarKey = "LC_ALL" + postgresLcAllEnvVarValue = "C" + + resultsDirname = "results" + stampFormat = time.RFC3339 + SqlStatsPrefix = "SQL statistics:" + read = "read" + write = "write" + other = "other" + totalQueries = "total" + totalEvents = "total number of events" + min = "min" + avg = "avg" + max = "max" + percentile = "percentile" + sum = "sum" + transactions = "transactions" + queriesPerSec = "queries" + ignoredErrors = "ignored errors" + reconnects = "reconnects" + totalTimeSecs = "total time" + + ResultFileTemplate = "%s_%s_%s_sysbench_performance%s" ) diff --git a/go/performance/utils/sysbench_runner/csv.go b/go/performance/utils/sysbench_runner/csv.go index 83e9eefa26c..44a445fe529 100644 --- a/go/performance/utils/sysbench_runner/csv.go +++ b/go/performance/utils/sysbench_runner/csv.go @@ -26,7 +26,7 @@ import ( // FromResultCsvHeaders returns supported csv headers for a Result func FromResultCsvHeaders() []string { return []string{ - "id", + "id", // todo: replace with constants "suite_id", "test_id", "runtime_os", diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go index 5b03293d3a8..4aeb4ec9297 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -2,6 +2,7 @@ package sysbench_runner import ( "context" + "errors" "os" "path/filepath" "syscall" @@ -10,6 +11,8 @@ import ( "github.com/dolthub/dolt/go/store/types" ) +var ErrNotSysbenchTest = errors.New("sysbench test is required") + var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } type doltBenchmarkerImpl struct { @@ -82,7 +85,11 @@ func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { runs := b.config.GetRuns() for i := 0; i < runs; i++ { for _, test := range tests { - tester := NewSysbenchTester(b.config, b.serverConfig, test, serverParams, stampFunc) + t, ok := test.(SysbenchTest) + if !ok { + return nil, ErrNotSysbenchTest + } + tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc) r, err := tester.Test(ctx) if err != nil { server.Stop() diff --git a/go/performance/utils/sysbench_runner/dolt_server_config.go b/go/performance/utils/sysbench_runner/dolt_server_config.go index db3f8392bdc..b5358f43938 100644 --- a/go/performance/utils/sysbench_runner/dolt_server_config.go +++ b/go/performance/utils/sysbench_runner/dolt_server_config.go @@ -6,10 +6,6 @@ import ( "os" ) -const ( - CpuServerProfile = "cpu" -) - type ServerProfile string type doltServerConfigImpl struct { @@ -89,6 +85,10 @@ func (sc *doltServerConfigImpl) GetServerType() ServerType { return Dolt } +func (sc *doltServerConfigImpl) GetResultsFormat() string { + return sc.ResultsFormat +} + func (sc *doltServerConfigImpl) GetServerExec() string { return sc.ServerExec } @@ -107,18 +107,18 @@ func (sc *doltServerConfigImpl) GetServerArgs() ([]string, error) { return params, nil } -func (sc *doltServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { - params := make([]string, 0) - params = append(params, defaultSysbenchParams...) - params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) // todo: replace these with consts - params = append(params, "--db-driver=mysql") - params = append(params, fmt.Sprintf("--mysql-host=%s", sc.Host)) - params = append(params, "--mysql-user=root") +func (sc *doltServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams { + params := NewSysbenchTestParams() + params.Append(defaultSysbenchParams...) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlDbFlag, dbName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, mysqlDriverName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlHostFlag, sc.Host)) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlUserFlag, defaultMysqlUser)) if sc.Port != 0 { - params = append(params, fmt.Sprintf("--mysql-port=%d", sc.Port)) + params.Append(fmt.Sprintf("%s=%d", sysbenchMysqlPortFlag, sc.Port)) } - params = append(params, testConfig.GetOptions()...) - params = append(params, testConfig.GetName()) + params.Append(testConfig.GetOptions()...) + params.Append(testConfig.GetName()) return params } diff --git a/go/performance/utils/sysbench_runner/doltgres.go b/go/performance/utils/sysbench_runner/doltgres.go index daeb7830772..b2dc6e9e08b 100644 --- a/go/performance/utils/sysbench_runner/doltgres.go +++ b/go/performance/utils/sysbench_runner/doltgres.go @@ -9,14 +9,6 @@ import ( "syscall" ) -const ( - postgresDriver = "postgres" - doltgresUser = "doltgres" - doltDataDir = ".dolt" - createDatabaseTemplate = "create database %s;" - psqlDsnTemplate = "host=%s port=%d user=%s password=%s dbname=%s sslmode=disable" -) - type doltgresBenchmarkerImpl struct { dir string // cwd config SysbenchConfig @@ -126,7 +118,11 @@ func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Result runs := b.config.GetRuns() for i := 0; i < runs; i++ { for _, test := range tests { - tester := NewSysbenchTester(b.config, b.serverConfig, test, serverParams, stampFunc) + t, ok := test.(SysbenchTest) + if !ok { + return nil, ErrNotSysbenchTest + } + tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc) var r *Result r, err = tester.Test(ctx) if err != nil { diff --git a/go/performance/utils/sysbench_runner/doltgres_server_config.go b/go/performance/utils/sysbench_runner/doltgres_server_config.go index 505ebd87b53..cd9ff3a3fba 100644 --- a/go/performance/utils/sysbench_runner/doltgres_server_config.go +++ b/go/performance/utils/sysbench_runner/doltgres_server_config.go @@ -54,6 +54,10 @@ func (sc *doltgresServerConfigImpl) GetServerExec() string { return sc.ServerExec } +func (sc *doltgresServerConfigImpl) GetResultsFormat() string { + return sc.ResultsFormat +} + func (sc *doltgresServerConfigImpl) GetServerArgs() ([]string, error) { params := make([]string, 0) if sc.Host != "" { @@ -66,18 +70,18 @@ func (sc *doltgresServerConfigImpl) GetServerArgs() ([]string, error) { return params, nil } -func (sc *doltgresServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { - params := make([]string, 0) - params = append(params, defaultSysbenchParams...) - params = append(params, "--db-driver=pgsql") // todo: replace with consts - params = append(params, fmt.Sprintf("--pgsql-db=%s", dbName)) - params = append(params, fmt.Sprintf("--pgsql-host=%s", sc.Host)) - params = append(params, "--pgsql-user=doltgres") +func (sc *doltgresServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams { + params := NewSysbenchTestParams() + params.Append(defaultSysbenchParams...) + params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, sysbenchPostgresDbDriver)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresDbFlag, dbName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresHostFlag, sc.Host)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresUserFlag, doltgresUser)) if sc.Port != 0 { - params = append(params, fmt.Sprintf("--pgsql-port=%d", sc.Port)) + params.Append(fmt.Sprintf("%s=%d", sysbenchPostgresPortFlag, sc.Port)) } - params = append(params, testConfig.GetOptions()...) - params = append(params, testConfig.GetName()) + params.Append(testConfig.GetOptions()...) + params.Append(testConfig.GetName()) return params } diff --git a/go/performance/utils/sysbench_runner/mysql.go b/go/performance/utils/sysbench_runner/mysql.go index 0907857a75e..1fa732a33ea 100644 --- a/go/performance/utils/sysbench_runner/mysql.go +++ b/go/performance/utils/sysbench_runner/mysql.go @@ -70,7 +70,11 @@ func (b *mysqlBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { runs := b.config.GetRuns() for i := 0; i < runs; i++ { for _, test := range tests { - tester := NewSysbenchTester(b.config, b.serverConfig, test, serverParams, stampFunc) + t, ok := test.(SysbenchTest) + if !ok { + return nil, ErrNotSysbenchTest + } + tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc) r, err := tester.Test(ctx) if err != nil { server.Stop() diff --git a/go/performance/utils/sysbench_runner/mysql_server_config.go b/go/performance/utils/sysbench_runner/mysql_server_config.go index 10f7f675e99..b5714a35d16 100644 --- a/go/performance/utils/sysbench_runner/mysql_server_config.go +++ b/go/performance/utils/sysbench_runner/mysql_server_config.go @@ -62,10 +62,30 @@ func (sc *mysqlServerConfigImpl) GetServerExec() string { return sc.ServerExec } +func (sc *mysqlServerConfigImpl) GetId() string { + return sc.Id +} + +func (sc *mysqlServerConfigImpl) GetHost() string { + return sc.Host +} + +func (sc *mysqlServerConfigImpl) GetPort() int { + return sc.Port +} + +func (sc *mysqlServerConfigImpl) GetVersion() string { + return sc.Version +} + func (sc *mysqlServerConfigImpl) GetServerType() ServerType { return MySql } +func (sc *mysqlServerConfigImpl) GetResultsFormat() string { + return sc.ResultsFormat +} + func (sc *mysqlServerConfigImpl) GetServerArgs() ([]string, error) { params := make([]string, 0) if sc.ServerUser != "" { @@ -81,19 +101,19 @@ func (sc *mysqlServerConfigImpl) GetServerArgs() ([]string, error) { return params, nil } -func (sc *mysqlServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { - params := make([]string, 0) - params = append(params, defaultSysbenchParams...) - params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) // todo: replace with consts - params = append(params, "--db-driver=mysql") - params = append(params, fmt.Sprintf("--mysql-host=%s", sc.Host)) +func (sc *mysqlServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams { + params := NewSysbenchTestParams() + params.Append(defaultSysbenchParams...) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlDbFlag, dbName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, mysqlDriverName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlHostFlag, sc.Host)) if sc.Port != 0 { - params = append(params, fmt.Sprintf("--mysql-port=%d", sc.Port)) + params.Append(fmt.Sprintf("%s=%d", sysbenchMysqlPortFlag, sc.Port)) } - params = append(params, "--mysql-user=sysbench") - params = append(params, fmt.Sprintf("--mysql-password=%s", sysbenchPassLocal)) - params = append(params, testConfig.GetOptions()...) - params = append(params, testConfig.GetName()) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlUserFlag, sysbenchCommand)) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlPasswordFlag, sysbenchPassLocal)) + params.Append(testConfig.GetOptions()...) + params.Append(testConfig.GetName()) return params } @@ -123,19 +143,3 @@ func (sc *mysqlServerConfigImpl) SetDefaults() error { } return nil } - -func (sc *mysqlServerConfigImpl) GetId() string { - return sc.Id -} - -func (sc *mysqlServerConfigImpl) GetHost() string { - return sc.Host -} - -func (sc *mysqlServerConfigImpl) GetPort() int { - return sc.Port -} - -func (sc *mysqlServerConfigImpl) GetVersion() string { - return sc.Version -} diff --git a/go/performance/utils/sysbench_runner/mysql_tpcc.go b/go/performance/utils/sysbench_runner/mysql_tpcc.go index 87fb14c9602..ad88919db4e 100644 --- a/go/performance/utils/sysbench_runner/mysql_tpcc.go +++ b/go/performance/utils/sysbench_runner/mysql_tpcc.go @@ -9,13 +9,13 @@ import ( type mysqlTpccBenchmarkerImpl struct { dir string // cwd - config *TpccBenchmarkConfig - serverConfig *doltServerConfigImpl + config TpccConfig + serverConfig ProtocolServerConfig } var _ Benchmarker = &mysqlTpccBenchmarkerImpl{} -func NewMysqlTpccBenchmarker(dir string, config *TpccBenchmarkConfig, serverConfig *doltServerConfigImpl) *mysqlTpccBenchmarkerImpl { +func NewMysqlTpccBenchmarker(dir string, config TpccConfig, serverConfig ProtocolServerConfig) *mysqlTpccBenchmarkerImpl { return &mysqlTpccBenchmarkerImpl{ dir: dir, config: config, @@ -24,7 +24,7 @@ func NewMysqlTpccBenchmarker(dir string, config *TpccBenchmarkConfig, serverConf } func (b *mysqlTpccBenchmarkerImpl) getDsn() (string, error) { - return GetMysqlDsn(b.serverConfig.Host, b.serverConfig.Socket, b.serverConfig.ConnectionProtocol, b.serverConfig.Port) + return GetMysqlDsn(b.serverConfig.GetHost(), b.serverConfig.GetSocket(), b.serverConfig.GetConnectionProtocol(), b.serverConfig.GetPort()) } func (b *mysqlTpccBenchmarkerImpl) createTestingDb(ctx context.Context) error { @@ -36,7 +36,7 @@ func (b *mysqlTpccBenchmarkerImpl) createTestingDb(ctx context.Context) error { } func (b *mysqlTpccBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { - serverDir, err := InitMysqlDataDir(ctx, b.serverConfig.ServerExec, tpccDbName) + serverDir, err := InitMysqlDataDir(ctx, b.serverConfig.GetServerExec(), tpccDbName) if err != nil { return nil, err } diff --git a/go/performance/utils/sysbench_runner/postgres.go b/go/performance/utils/sysbench_runner/postgres.go index 264a70adec5..c976c9cda85 100644 --- a/go/performance/utils/sysbench_runner/postgres.go +++ b/go/performance/utils/sysbench_runner/postgres.go @@ -10,28 +10,15 @@ import ( _ "github.com/lib/pq" ) -const ( - postgresInitDbDataDirFlag = "--pgdata" - postgresUsernameFlag = "--username" - postgresUsername = "postgres" - postgresDataDirFlag = "-D" - postgresDropDatabaseSqlTemplate = "DROP DATABASE IF EXISTS %s;" - postgresDropUserSqlTemplate = "DROP USER IF EXISTS %s;" - postgresCreateUserSqlTemplate = "CREATE USER %s WITH PASSWORD '%s';" - postgresCreateDatabaseSqlTemplate = "CREATE DATABASE %s WITH OWNER %s;" - postgresLcAllEnvVarKey = "LC_ALL" - postgresLcAllEnvVarValue = "C" -) - type postgresBenchmarkerImpl struct { dir string // cwd - config *sysbenchRunnerConfigImpl - serverConfig *doltServerConfigImpl + config SysbenchConfig + serverConfig InitServerConfig } var _ Benchmarker = &postgresBenchmarkerImpl{} -func NewPostgresBenchmarker(dir string, config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl) *postgresBenchmarkerImpl { +func NewPostgresBenchmarker(dir string, config SysbenchConfig, serverConfig InitServerConfig) *postgresBenchmarkerImpl { return &postgresBenchmarkerImpl{ dir: dir, config: config, @@ -45,7 +32,7 @@ func (b *postgresBenchmarkerImpl) initDataDir(ctx context.Context) (string, erro return "", err } - pgInit := ExecCommand(ctx, b.serverConfig.InitExec, fmt.Sprintf("%s=%s", postgresInitDbDataDirFlag, serverDir), fmt.Sprintf("%s=%s", postgresUsernameFlag, postgresUsername)) + pgInit := ExecCommand(ctx, b.serverConfig.GetInitDbExec(), fmt.Sprintf("%s=%s", postgresInitDbDataDirFlag, serverDir), fmt.Sprintf("%s=%s", postgresUsernameFlag, postgresUsername)) err = pgInit.Run() if err != nil { return "", err @@ -55,7 +42,7 @@ func (b *postgresBenchmarkerImpl) initDataDir(ctx context.Context) (string, erro } func (b *postgresBenchmarkerImpl) createTestingDb(ctx context.Context) (err error) { - psqlconn := fmt.Sprintf(psqlDsnTemplate, b.serverConfig.Host, b.serverConfig.Port, postgresUsername, "", dbName) + psqlconn := fmt.Sprintf(psqlDsnTemplate, b.serverConfig.GetHost(), b.serverConfig.GetPort(), postgresUsername, "", dbName) var db *sql.DB db, err = sql.Open(postgresDriver, psqlconn) @@ -124,16 +111,21 @@ func (b *postgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Result return } - var tests []*sysbenchTestImpl - tests, err = GetTests(b.config, b.serverConfig, nil) + var tests []Test + tests, err = GetTests(b.config, b.serverConfig) if err != nil { return } results = make(Results, 0) - for i := 0; i < b.config.Runs; i++ { + runs := b.config.GetRuns() + for i := 0; i < runs; i++ { for _, test := range tests { - tester := NewSysbenchTester(b.config, b.serverConfig, test, serverParams, stampFunc) + t, ok := test.(SysbenchTest) + if !ok { + return nil, ErrNotSysbenchTest + } + tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc) var r *Result r, err = tester.Test(ctx) if err != nil { diff --git a/go/performance/utils/sysbench_runner/postgres_server_config.go b/go/performance/utils/sysbench_runner/postgres_server_config.go index 955685bd895..e1e790f2e3f 100644 --- a/go/performance/utils/sysbench_runner/postgres_server_config.go +++ b/go/performance/utils/sysbench_runner/postgres_server_config.go @@ -58,6 +58,26 @@ func (sc *postgresServerConfigImpl) GetInitDbExec() string { return sc.InitExec } +func (sc *postgresServerConfigImpl) GetId() string { + return sc.Id +} + +func (sc *postgresServerConfigImpl) GetHost() string { + return sc.Host +} + +func (sc *postgresServerConfigImpl) GetPort() int { + return sc.Port +} + +func (sc *postgresServerConfigImpl) GetVersion() string { + return sc.Version +} + +func (sc *postgresServerConfigImpl) GetResultsFormat() string { + return sc.ResultsFormat +} + func (sc *postgresServerConfigImpl) GetServerType() ServerType { return Postgres } @@ -71,18 +91,18 @@ func (sc *postgresServerConfigImpl) GetServerArgs() ([]string, error) { return params, nil } -func (sc *postgresServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { - params := make([]string, 0) - params = append(params, defaultSysbenchParams...) - params = append(params, "--db-driver=pgsql") - params = append(params, fmt.Sprintf("--pgsql-db=%s", dbName)) - params = append(params, fmt.Sprintf("--pgsql-host=%s", sc.Host)) - params = append(params, "--pgsql-user=postgres") +func (sc *postgresServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams { + params := NewSysbenchTestParams() + params.Append(defaultSysbenchParams...) + params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, sysbenchPostgresDbDriver)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresDbFlag, dbName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresHostFlag, sc.Host)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresUserFlag, postgresUsername)) if sc.Port != 0 { - params = append(params, fmt.Sprintf("--pgsql-port=%d", sc.Port)) + params.Append(fmt.Sprintf("%s=%d", sysbenchPostgresPortFlag, sc.Port)) } - params = append(params, testConfig.GetOptions()...) - params = append(params, testConfig.GetName()) + params.Append(testConfig.GetOptions()...) + params.Append(testConfig.GetName()) return params } @@ -112,15 +132,3 @@ func (sc *postgresServerConfigImpl) SetDefaults() error { } return nil } - -func (sc *postgresServerConfigImpl) GetId() string { - return sc.Id -} - -func (sc *postgresServerConfigImpl) GetHost() string { - return sc.Host -} - -func (sc *postgresServerConfigImpl) GetPort() int { - return sc.Port -} diff --git a/go/performance/utils/sysbench_runner/profile.go b/go/performance/utils/sysbench_runner/profile.go index ae0cdbb83fb..0a86a05ddfd 100644 --- a/go/performance/utils/sysbench_runner/profile.go +++ b/go/performance/utils/sysbench_runner/profile.go @@ -14,13 +14,13 @@ type Profiler interface { type doltProfilerImpl struct { dir string // cwd - config *sysbenchRunnerConfigImpl - serverConfig *doltServerConfigImpl + config SysbenchConfig + serverConfig ProfilingServerConfig } var _ Profiler = &doltProfilerImpl{} -func NewDoltProfiler(dir string, config *sysbenchRunnerConfigImpl, serverConfig *doltServerConfigImpl) *doltProfilerImpl { +func NewDoltProfiler(dir string, config SysbenchConfig, serverConfig ProfilingServerConfig) *doltProfilerImpl { return &doltProfilerImpl{ dir: dir, config: config, @@ -29,20 +29,20 @@ func NewDoltProfiler(dir string, config *sysbenchRunnerConfigImpl, serverConfig } func (p *doltProfilerImpl) updateGlobalConfig(ctx context.Context) error { - err := CheckSetDoltConfig(ctx, p.serverConfig.ServerExec, doltConfigUsernameKey, doltBenchmarkUser) + err := CheckSetDoltConfig(ctx, p.serverConfig.GetServerExec(), doltConfigUsernameKey, doltBenchmarkUser) if err != nil { return err } - return CheckSetDoltConfig(ctx, p.serverConfig.ServerExec, doltConfigEmailKey, doltBenchmarkEmail) + return CheckSetDoltConfig(ctx, p.serverConfig.GetServerExec(), doltConfigEmailKey, doltBenchmarkEmail) } func (p *doltProfilerImpl) checkInstallation(ctx context.Context) error { - version := ExecCommand(ctx, p.serverConfig.ServerExec, doltVersionCommand) + version := ExecCommand(ctx, p.serverConfig.GetServerExec(), doltVersionCommand) return version.Run() } func (p *doltProfilerImpl) initDoltRepo(ctx context.Context) (string, error) { - return InitDoltRepo(ctx, p.dir, p.serverConfig.ServerExec, p.config.NomsBinFormat, dbName) + return InitDoltRepo(ctx, p.dir, p.serverConfig.GetServerExec(), p.config.GetNomsBinFormat(), dbName) } func (p *doltProfilerImpl) Profile(ctx context.Context) error { @@ -84,15 +84,20 @@ func (p *doltProfilerImpl) Profile(ctx context.Context) error { return err } - tests, err := GetTests(p.config, p.serverConfig, nil) + tests, err := GetTests(p.config, p.serverConfig) if err != nil { return err } results := make(Results, 0) - for i := 0; i < p.config.Runs; i++ { + runs := p.config.GetRuns() + for i := 0; i < runs; i++ { for _, test := range tests { - tester := NewSysbenchTester(p.config, p.serverConfig, test, profileParams, stampFunc) + t, ok := test.(SysbenchTest) + if !ok { + return ErrNotSysbenchTest + } + tester := NewSysbenchTester(p.config, p.serverConfig, t, profileParams, stampFunc) r, err := tester.Test(ctx) if err != nil { server.Stop() @@ -116,6 +121,6 @@ func (p *doltProfilerImpl) Profile(ctx context.Context) error { return fmt.Errorf("failed to create profile: file was empty") } - finalProfile := filepath.Join(p.serverConfig.ProfilePath, fmt.Sprintf("%s_%s", p.serverConfig.Id, cpuProfileFilename)) + finalProfile := filepath.Join(p.serverConfig.GetProfilePath(), fmt.Sprintf("%s_%s", p.serverConfig.GetId(), cpuProfileFilename)) return os.Rename(tempProfile, finalProfile) } diff --git a/go/performance/utils/sysbench_runner/results.go b/go/performance/utils/sysbench_runner/results.go index 99029ddff68..10ab10d6e52 100644 --- a/go/performance/utils/sysbench_runner/results.go +++ b/go/performance/utils/sysbench_runner/results.go @@ -16,37 +16,13 @@ package sysbench_runner import ( "errors" + "github.com/google/uuid" "path/filepath" "strconv" "strings" - "time" - - "github.com/google/uuid" -) - -const ( - stampFormat = time.RFC3339 - SqlStatsPrefix = "SQL statistics:" - read = "read" - write = "write" - other = "other" - totalQueries = "total" - totalEvents = "total number of events" - min = "min" - avg = "avg" - max = "max" - percentile = "percentile" - sum = "sum" - transactions = "transactions" - queriesPerSec = "queries" - ignoredErrors = "ignored errors" - reconnects = "reconnects" - totalTimeSecs = "total time" ) var ( - ResultFileTemplate = "%s_%s_%s_sysbench_performance%s" - ErrUnableToParseOutput = errors.New("unable to parse output") ErrUnsupportedHeaderField = errors.New("unsupported header field") ) diff --git a/go/performance/utils/sysbench_runner/results_test.go b/go/performance/utils/sysbench_runner/results_test.go index 76db1f40f16..7dbbc9e0eb6 100644 --- a/go/performance/utils/sysbench_runner/results_test.go +++ b/go/performance/utils/sysbench_runner/results_test.go @@ -266,9 +266,9 @@ func TestFromOutputResults(t *testing.T) { tests := []struct { description string output []byte - config *sysbenchRunnerConfigImpl - serverConfig *doltServerConfigImpl - test *sysbenchTestImpl + config Config + serverConfig ServerConfig + test Test expectedResult *Result expectedError error }{ @@ -319,9 +319,60 @@ func TestFromOutputResults(t *testing.T) { t.Run(test.description, func(t *testing.T) { serverParams, err := test.serverConfig.GetServerArgs() assert.NoError(t, err) - actual, err := OutputToResult(test.output, test.serverConfig.Server, test.serverConfig.Version, test.test.Name, test.test.id, testSuiteId, test.config.RuntimeOS, test.config.RuntimeGoArch, serverParams, test.test.Params, func() string { return testId }, test.test.FromScript) + actual, err := OutputToResult(test.output, test.serverConfig.GetServerType(), test.serverConfig.GetVersion(), test.test.Name, test.test.id, testSuiteId, test.config.RuntimeOS, test.config.RuntimeGoArch, serverParams, test.test.Params, func() string { return testId }, test.test.FromScript) assert.Equal(t, test.expectedError, err) assert.Equal(t, test.expectedResult, actual) }) } } + +type testServerConfigImpl struct{} + +func (t testServerConfigImpl) GetId() string { + return "test-id" +} + +func (t testServerConfigImpl) GetHost() string { + return "test-host" +} + +func (t testServerConfigImpl) GetPort() int { + return 1234 +} + +func (t testServerConfigImpl) GetVersion() string { + return "test-version" +} + +func (t testServerConfigImpl) GetServerExec() string { + return "test-server-exec" +} + +func (t testServerConfigImpl) GetResultsFormat() string { + return CsvFormat +} + +func (t testServerConfigImpl) GetServerType() ServerType { + return ServerType(testServer) +} + +func (t testServerConfigImpl) GetServerArgs() ([]string, error) { + return []string{}, nil +} + +func (t testServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { + //TODO implement me + panic("implement me") +} + +func (t testServerConfigImpl) Validate() error { + //TODO implement me + panic("implement me") +} + +func (t testServerConfigImpl) SetDefaults() error { + //TODO implement me + panic("implement me") +} + +var _ ServerConfig = &testServerConfigImpl{} diff --git a/go/performance/utils/sysbench_runner/run.go b/go/performance/utils/sysbench_runner/run.go index 551cd5939a3..87426b56e8b 100644 --- a/go/performance/utils/sysbench_runner/run.go +++ b/go/performance/utils/sysbench_runner/run.go @@ -16,14 +16,18 @@ package sysbench_runner import ( "context" + "errors" "fmt" "os" "path/filepath" ) +var ErrNotProtocolServerConfig = errors.New("protocol server config required") +var ErrNotInitDbServerConfig = errors.New("init db server config required") + // Run runs sysbench runner -func Run(ctx context.Context, config *sysbenchRunnerConfigImpl) error { - err := config.Validate() +func Run(ctx context.Context, config SysbenchConfig) error { + err := config.Validate(ctx) if err != nil { return err } @@ -38,30 +42,44 @@ func Run(ctx context.Context, config *sysbenchRunnerConfigImpl) error { return err } - for _, serverConfig := range config.Servers { + svs := config.GetServerConfigs() + for _, serverConfig := range svs { var results Results var b Benchmarker - switch serverConfig.Server { + st := serverConfig.GetServerType() + switch st { case Dolt: // handle a profiling run - if serverConfig.ServerProfile != "" { - fmt.Println("Profiling dolt while running sysbench tests") - p := NewDoltProfiler(cwd, config, serverConfig) - return p.Profile(ctx) + sc, ok := serverConfig.(ProfilingServerConfig) + if ok { + if string(sc.GetServerProfile()) != "" { + fmt.Println("Profiling dolt while running sysbench tests") + p := NewDoltProfiler(cwd, config, sc) + return p.Profile(ctx) + } } + fmt.Println("Running dolt sysbench tests") b = NewDoltBenchmarker(cwd, config, serverConfig) case Doltgres: fmt.Println("Running doltgres sysbench tests") b = NewDoltgresBenchmarker(cwd, config, serverConfig) case MySql: + sc, ok := serverConfig.(ProtocolServerConfig) + if !ok { + return ErrNotProtocolServerConfig + } fmt.Println("Running mysql sysbench tests") - b = NewMysqlBenchmarker(cwd, config, serverConfig) + b = NewMysqlBenchmarker(cwd, config, sc) case Postgres: + sc, ok := serverConfig.(InitServerConfig) + if !ok { + return ErrNotInitDbServerConfig + } fmt.Println("Running postgres sysbench tests") - b = NewPostgresBenchmarker(cwd, config, serverConfig) + b = NewPostgresBenchmarker(cwd, config, sc) default: - panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server)) + panic(fmt.Sprintf("unexpected server type: %s", st)) } results, err = b.Benchmark(ctx) @@ -69,49 +87,55 @@ func Run(ctx context.Context, config *sysbenchRunnerConfigImpl) error { return err } - fmt.Printf("Successfuly finished %s\n", serverConfig.Server) + fmt.Printf("Successfuly finished %s\n", st) err = WriteResults(serverConfig, results) if err != nil { return err } - fmt.Printf("Successfuly wrote results for %s\n", serverConfig.Server) + fmt.Printf("Successfuly wrote results for %s\n", st) } return nil } func sysbenchVersion(ctx context.Context) error { - sysbenchVersion := ExecCommand(ctx, "sysbench", "--version") - return sysbenchVersion.Run() + version := ExecCommand(ctx, sysbenchCommand, sysbenchVersionFlag) + return version.Run() } -func WriteResults(serverConfig *doltServerConfigImpl, results Results) error { +func WriteResults(serverConfig ServerConfig, results Results) error { + st := serverConfig.GetServerType() + version := serverConfig.GetVersion() + id := serverConfig.GetId() + format := serverConfig.GetResultsFormat() + cwd, err := os.Getwd() if err != nil { return err } - var writePath string - switch serverConfig.ResultsFormat { + + //var writePath string + switch format { case CsvFormat, CsvExt: - writePath = filepath.Join( + writePath := filepath.Join( cwd, - "results", - string(serverConfig.Server), - serverConfig.Version, - serverConfig.GetId(), - fmt.Sprintf(ResultFileTemplate, serverConfig.GetId(), serverConfig.Server, serverConfig.Version, CsvExt)) + resultsDirname, + string(st), + version, + id, + fmt.Sprintf(ResultFileTemplate, id, st, version, CsvExt)) return WriteResultsCsv(writePath, results) case JsonFormat, JsonExt: - writePath = filepath.Join( + writePath := filepath.Join( cwd, - "results", - string(serverConfig.Server), - serverConfig.Version, - serverConfig.GetId(), - fmt.Sprintf(ResultFileTemplate, serverConfig.GetId(), serverConfig.Server, serverConfig.Version, JsonExt)) + resultsDirname, + string(st), + version, + id, + fmt.Sprintf(ResultFileTemplate, id, st, version, JsonExt)) return WriteResultsJson(writePath, results) default: + return fmt.Errorf("unsupported results format: %s", format) } - return fmt.Errorf("unsupported results format: %s", serverConfig.ResultsFormat) } diff --git a/go/performance/utils/sysbench_runner/run_tpcc.go b/go/performance/utils/sysbench_runner/run_tpcc.go index 96f0c168b0f..142a6cd4feb 100644 --- a/go/performance/utils/sysbench_runner/run_tpcc.go +++ b/go/performance/utils/sysbench_runner/run_tpcc.go @@ -20,8 +20,8 @@ import ( "os" ) -func RunTpcc(ctx context.Context, config *TpccBenchmarkConfig) error { - err := config.updateDefaults() +func RunTpcc(ctx context.Context, config TpccConfig) error { + err := config.Validate(ctx) if err != nil { return err } @@ -31,18 +31,25 @@ func RunTpcc(ctx context.Context, config *TpccBenchmarkConfig) error { return err } - for _, serverConfig := range config.Servers { + svs := config.GetServerConfigs() + for _, serverConfig := range svs { var b Benchmarker var results Results - switch serverConfig.Server { + st := serverConfig.GetServerType() + switch st { case Dolt: fmt.Println("Running dolt tpcc benchmarks") b = NewDoltTpccBenchmarker(cwd, config, serverConfig) case MySql: + sc, ok := serverConfig.(ProtocolServerConfig) + if !ok { + return ErrNotProtocolServerConfig + } + fmt.Println("Running mysql tpcc benchmarks") - b = NewMysqlTpccBenchmarker(cwd, config, serverConfig) + b = NewMysqlTpccBenchmarker(cwd, config, sc) default: - panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server)) + panic(fmt.Sprintf("unexpected server type: %s", st)) } results, err = b.Benchmark(ctx) @@ -55,7 +62,7 @@ func RunTpcc(ctx context.Context, config *TpccBenchmarkConfig) error { return err } - fmt.Printf("Successfuly wrote results for %s\n", serverConfig.Server) + fmt.Printf("Successfuly wrote results for %s\n", st) } return nil diff --git a/go/performance/utils/sysbench_runner/server.go b/go/performance/utils/sysbench_runner/server.go index 555b26eaad9..ee0aca40977 100644 --- a/go/performance/utils/sysbench_runner/server.go +++ b/go/performance/utils/sysbench_runner/server.go @@ -36,7 +36,7 @@ func NewServer(ctx context.Context, dir string, serverConfig ServerConfig, killS withKeyCtx, cancel := context.WithCancel(ctx) gServer, serverCtx := errgroup.WithContext(withKeyCtx) - server := ExecCommand(serverCtx, serverConfig.ServerExec(), serverParams...) + server := ExecCommand(serverCtx, serverConfig.GetServerExec(), serverParams...) server.Dir = dir quit := make(chan os.Signal, 1) diff --git a/go/performance/utils/sysbench_runner/server_config.go b/go/performance/utils/sysbench_runner/server_config.go index 7a5d3c29633..a1581ac74cf 100644 --- a/go/performance/utils/sysbench_runner/server_config.go +++ b/go/performance/utils/sysbench_runner/server_config.go @@ -8,9 +8,10 @@ type ServerConfig interface { GetPort() int GetVersion() string GetServerExec() string + GetResultsFormat() string GetServerType() ServerType GetServerArgs() ([]string, error) - GetTestingArgs(testConfig TestConfig) []string + GetTestingParams(testConfig TestConfig) TestParams Validate() error SetDefaults() error } diff --git a/go/performance/utils/sysbench_runner/sysbench.go b/go/performance/utils/sysbench_runner/sysbench.go index 63b1f1166e6..69b1eccd3c1 100644 --- a/go/performance/utils/sysbench_runner/sysbench.go +++ b/go/performance/utils/sysbench_runner/sysbench.go @@ -12,7 +12,7 @@ import ( ) type sysbenchTesterImpl struct { - test Test + test SysbenchTest config Config serverConfig ServerConfig serverParams []string @@ -23,19 +23,19 @@ type sysbenchTesterImpl struct { var _ Tester = &sysbenchTesterImpl{} -func NewSysbenchTester(config Config, serverConfig ServerConfig, test Test, serverParams []string, stampFunc func() string) *sysbenchTesterImpl { +func NewSysbenchTester(config Config, serverConfig ServerConfig, test SysbenchTest, serverParams []string, stampFunc func() string) *sysbenchTesterImpl { return &sysbenchTesterImpl{ config: config, serverParams: serverParams, serverConfig: serverConfig, test: test, - suiteId: serverConfig.Id(), + suiteId: serverConfig.GetId(), stampFunc: stampFunc, } } func (t *sysbenchTesterImpl) newResult() (*Result, error) { - serverParams, err := t.serverConfig.ServerArgs() + serverParams, err := t.serverConfig.GetServerArgs() if err != nil { return nil, err } @@ -50,36 +50,36 @@ func (t *sysbenchTesterImpl) newResult() (*Result, error) { } var name string - if t.test.FromScript() { - base := filepath.Base(t.test.Name()) + if t.test.GetFromScript() { + base := filepath.Base(t.test.GetName()) ext := filepath.Ext(base) name = strings.TrimSuffix(base, ext) } else { - name = t.test.Name() + name = t.test.GetName() } return &Result{ Id: getId(), SuiteId: t.suiteId, - TestId: t.test.Id(), - RuntimeOS: t.config.RuntimeOs(), - RuntimeGoArch: t.config.RuntimeGoArch(), - ServerName: string(t.serverConfig.ServerType()), - ServerVersion: t.serverConfig.Version(), + TestId: t.test.GetId(), + RuntimeOS: t.config.GetRuntimeOs(), + RuntimeGoArch: t.config.GetRuntimeGoArch(), + ServerName: string(t.serverConfig.GetServerType()), + ServerVersion: t.serverConfig.GetVersion(), ServerParams: strings.Join(serverParams, " "), TestName: name, - TestParams: strings.Join(t.test.ParamsToSlice(), " "), + TestParams: strings.Join(t.test.GetParamsToSlice(), " "), }, nil } func (t *sysbenchTesterImpl) outputToResult(output []byte) (*Result, error) { - return OutputToResult(output, t.serverConfig.ServerType(), t.serverConfig.Version(), t.test.Name(), t.test.Id(), t.suiteId, t.config.RuntimeOs(), t.config.RuntimeGoArch(), t.serverParams, t.test.ParamsToSlice(), nil, t.test.FromScript()) + return OutputToResult(output, t.serverConfig.GetServerType(), t.serverConfig.GetVersion(), t.test.GetName(), t.test.GetId(), t.suiteId, t.config.GetRuntimeOs(), t.config.GetRuntimeGoArch(), t.serverParams, t.test.GetParamsToSlice(), nil, t.test.GetFromScript()) } func (t *sysbenchTesterImpl) prepare(ctx context.Context) error { - cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.PrepareArgs()...) - if t.test.FromScript() { - lp := filepath.Join(t.config.ScriptDir(), luaPath) + cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.GetPrepareArgs(t.serverConfig)...) + if t.test.GetFromScript() { + lp := filepath.Join(t.config.GetScriptDir(), luaPath) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) } @@ -92,9 +92,9 @@ func (t *sysbenchTesterImpl) prepare(ctx context.Context) error { } func (t *sysbenchTesterImpl) run(ctx context.Context) (*Result, error) { - cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.RunArgs()...) - if t.test.FromScript() { - lp := filepath.Join(t.config.ScriptDir(), luaPath) + cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.GetRunArgs(t.serverConfig)...) + if t.test.GetFromScript() { + lp := filepath.Join(t.config.GetScriptDir(), luaPath) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) } @@ -120,9 +120,9 @@ func (t *sysbenchTesterImpl) run(ctx context.Context) (*Result, error) { } func (t *sysbenchTesterImpl) cleanup(ctx context.Context) error { - cmd := ExecCommand(ctx, sysbenchCommand, t.test.CleanupArgs()...) - if t.test.FromScript() { - lp := filepath.Join(t.config.ScriptDir(), luaPath) + cmd := ExecCommand(ctx, sysbenchCommand, t.test.GetCleanupArgs(t.serverConfig)...) + if t.test.GetFromScript() { + lp := filepath.Join(t.config.GetScriptDir(), luaPath) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) } @@ -135,7 +135,7 @@ func (t *sysbenchTesterImpl) Test(ctx context.Context) (*Result, error) { return nil, err } - fmt.Println("Running test", t.test.Name) + fmt.Println("Running test", t.test.GetName()) rs, err := t.run(ctx) if err != nil { diff --git a/go/performance/utils/sysbench_runner/sysbench_config.go b/go/performance/utils/sysbench_runner/sysbench_config.go index 4c5fb23ad26..da5af42bfaf 100644 --- a/go/performance/utils/sysbench_runner/sysbench_config.go +++ b/go/performance/utils/sysbench_runner/sysbench_config.go @@ -138,6 +138,10 @@ func (c *sysbenchRunnerConfigImpl) GetTestOptions() []string { return c.TestOptions } +func (c *sysbenchRunnerConfigImpl) GetServerConfigs() []ServerConfig { + return c.Servers +} + // Validate checks the config for the required fields and sets defaults // where necessary func (c *sysbenchRunnerConfigImpl) Validate(ctx context.Context) error { diff --git a/go/performance/utils/sysbench_runner/sysbench_tests.go b/go/performance/utils/sysbench_runner/sysbench_tests.go index 39de6509156..723b593aa96 100644 --- a/go/performance/utils/sysbench_runner/sysbench_tests.go +++ b/go/performance/utils/sysbench_runner/sysbench_tests.go @@ -1,15 +1,22 @@ package sysbench_runner -type sysbenchTestParamsImpl []string +type sysbenchTestParamsImpl struct { + params []string +} + +var _ SysbenchTestParams = &sysbenchTestParamsImpl{} func (s *sysbenchTestParamsImpl) ToSlice() []string { - if s != nil { - return *s - } - return []string{} + return s.params } -var _ TestParams = &sysbenchTestParamsImpl{} +func (s *sysbenchTestParamsImpl) Append(params ...string) { + s.params = append(s.params, params...) +} + +func NewSysbenchTestParams() *sysbenchTestParamsImpl { + return &sysbenchTestParamsImpl{params: make([]string, 0)} +} // sysbenchTestImpl is a single sysbench test type sysbenchTestImpl struct { diff --git a/go/performance/utils/sysbench_runner/test_config.go b/go/performance/utils/sysbench_runner/test_config.go index a2e81242989..6e23e2acf81 100644 --- a/go/performance/utils/sysbench_runner/test_config.go +++ b/go/performance/utils/sysbench_runner/test_config.go @@ -61,13 +61,13 @@ func (ct *testConfigImpl) GetTests(serverConfig ServerConfig) ([]Test, error) { ct.N = 1 } - params := serverConfig.GetTestingArgs(ct) + params := serverConfig.GetTestingParams(ct) tests := make([]Test, 0) for i := 0; i < ct.N; i++ { - p := make([]string, len(params)) - copy(p, params) - tests = append(tests, NewSysbenchTest(ct.NewId(), ct.Name, p, ct.FromScript)) + //p := make([]string, params.Len()) + //copy(p, params) + tests = append(tests, NewSysbenchTest(ct.NewId(), ct.Name, params, ct.FromScript)) } return tests, nil diff --git a/go/performance/utils/sysbench_runner/tester.go b/go/performance/utils/sysbench_runner/tester.go index bb104a9d702..faba21e1937 100644 --- a/go/performance/utils/sysbench_runner/tester.go +++ b/go/performance/utils/sysbench_runner/tester.go @@ -24,6 +24,11 @@ type TestParams interface { ToSlice() []string } +type SysbenchTestParams interface { + TestParams + Append(params ...string) +} + type TpccTestParams interface { TestParams GetNumThreads() int diff --git a/go/performance/utils/sysbench_runner/tpcc_config.go b/go/performance/utils/sysbench_runner/tpcc_config.go index 0165754d31b..5d7c4445117 100644 --- a/go/performance/utils/sysbench_runner/tpcc_config.go +++ b/go/performance/utils/sysbench_runner/tpcc_config.go @@ -24,7 +24,7 @@ import ( var defaultTpccParams = []string{ fmt.Sprintf("%s=%s", tpccMysqlDbFlag, tpccDbName), - fmt.Sprintf("%s=%s", tpccDbDriverFlag, mysqlDriver), + fmt.Sprintf("%s=%s", tpccDbDriverFlag, mysqlDriverName), } // tpccConfigImpl represents a configuration for an execution of the TPCC Benchmark. It executes a series of tests @@ -91,6 +91,10 @@ func (c *tpccConfigImpl) GetScaleFactors() []int { return c.ScaleFactors } +func (c *tpccConfigImpl) GetServerConfigs() []ServerConfig { + return c.Servers +} + func (c *tpccConfigImpl) setDefaults() { // TODO: Eventually we need to support scale factors all the way to 10 if len(c.ScaleFactors) == 0 { diff --git a/go/performance/utils/sysbench_runner/tpcc_tests.go b/go/performance/utils/sysbench_runner/tpcc_tests.go index 9bdfe1e96b5..f8d7fdda334 100644 --- a/go/performance/utils/sysbench_runner/tpcc_tests.go +++ b/go/performance/utils/sysbench_runner/tpcc_tests.go @@ -92,13 +92,13 @@ type tpccTestImpl struct { Name string // Params are associated parameters this test runs with - Params TestParams + Params TpccTestParams } var _ Test = &tpccTestImpl{} // NewTpccTest instantiates and returns a TPCC test. -func NewTpccTest(name string, params TestParams) *tpccTestImpl { +func NewTpccTest(name string, params TpccTestParams) *tpccTestImpl { return &tpccTestImpl{ Id: uuid.New().String(), Name: name, From 46bc6bacabc805c394b9688e1f4c05e10cebd788 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Tue, 13 Feb 2024 14:55:53 -0800 Subject: [PATCH 18/25] /go/performance/utils/sysbench_runner: wip, need to test now --- .../utils/sysbench_runner/results_test.go | 61 ++---------------- .../utils/sysbench_runner/run_test.go | 11 ++-- .../sysbench_runner/sysbench_config_test.go | 63 ++++++++++--------- .../utils/sysbench_runner/sysbench_tests.go | 6 +- 4 files changed, 48 insertions(+), 93 deletions(-) diff --git a/go/performance/utils/sysbench_runner/results_test.go b/go/performance/utils/sysbench_runner/results_test.go index 7dbbc9e0eb6..795c2333beb 100644 --- a/go/performance/utils/sysbench_runner/results_test.go +++ b/go/performance/utils/sysbench_runner/results_test.go @@ -263,12 +263,15 @@ func fromResultSysbenchOutput(r *Result) string { } func TestFromOutputResults(t *testing.T) { + params := NewSysbenchTestParams() + params.Append(testTestParams) + tests := []struct { description string output []byte config Config serverConfig ServerConfig - test Test + test SysbenchTest expectedResult *Result expectedError error }{ @@ -281,13 +284,12 @@ func TestFromOutputResults(t *testing.T) { }, serverConfig: &doltServerConfigImpl{ Host: "localhost", - Server: ServerType(testServer), Version: testServerVersion, ServerExec: "test-exec", }, test: &sysbenchTestImpl{ Name: testTestName, - Params: []string{testTestParams}, + Params: params, }, expectedResult: &Result{ Id: testId, @@ -319,60 +321,9 @@ func TestFromOutputResults(t *testing.T) { t.Run(test.description, func(t *testing.T) { serverParams, err := test.serverConfig.GetServerArgs() assert.NoError(t, err) - actual, err := OutputToResult(test.output, test.serverConfig.GetServerType(), test.serverConfig.GetVersion(), test.test.Name, test.test.id, testSuiteId, test.config.RuntimeOS, test.config.RuntimeGoArch, serverParams, test.test.Params, func() string { return testId }, test.test.FromScript) + actual, err := OutputToResult(test.output, test.serverConfig.GetServerType(), test.serverConfig.GetVersion(), test.test.GetName(), test.test.GetId(), testSuiteId, test.config.GetRuntimeOs(), test.config.GetRuntimeGoArch(), serverParams, test.test.GetParamsToSlice(), func() string { return testId }, test.test.GetFromScript()) assert.Equal(t, test.expectedError, err) assert.Equal(t, test.expectedResult, actual) }) } } - -type testServerConfigImpl struct{} - -func (t testServerConfigImpl) GetId() string { - return "test-id" -} - -func (t testServerConfigImpl) GetHost() string { - return "test-host" -} - -func (t testServerConfigImpl) GetPort() int { - return 1234 -} - -func (t testServerConfigImpl) GetVersion() string { - return "test-version" -} - -func (t testServerConfigImpl) GetServerExec() string { - return "test-server-exec" -} - -func (t testServerConfigImpl) GetResultsFormat() string { - return CsvFormat -} - -func (t testServerConfigImpl) GetServerType() ServerType { - return ServerType(testServer) -} - -func (t testServerConfigImpl) GetServerArgs() ([]string, error) { - return []string{}, nil -} - -func (t testServerConfigImpl) GetTestingArgs(testConfig TestConfig) []string { - //TODO implement me - panic("implement me") -} - -func (t testServerConfigImpl) Validate() error { - //TODO implement me - panic("implement me") -} - -func (t testServerConfigImpl) SetDefaults() error { - //TODO implement me - panic("implement me") -} - -var _ ServerConfig = &testServerConfigImpl{} diff --git a/go/performance/utils/sysbench_runner/run_test.go b/go/performance/utils/sysbench_runner/run_test.go index 206b8328535..ffc24a61881 100644 --- a/go/performance/utils/sysbench_runner/run_test.go +++ b/go/performance/utils/sysbench_runner/run_test.go @@ -33,10 +33,9 @@ func TestRunner(t *testing.T) { conf := &sysbenchRunnerConfigImpl{ Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_delete_insert"), //Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_update_non_index", "oltp_insert", "bulk_insert", "oltp_write_only", "oltp_delete"), - Servers: []*doltServerConfigImpl{ - { + Servers: []ServerConfig{ + &doltServerConfigImpl{ Id: "test", - Server: Dolt, Version: "0.39.2", ResultsFormat: CsvFormat, ServerExec: "/Users/max-hoffman/go/bin/dolt", @@ -59,10 +58,10 @@ func TestRunner(t *testing.T) { } } -func selectTests(names ...string) []*TestConfigImpl { - tests := make([]*TestConfigImpl, len(names)) +func selectTests(names ...string) []TestConfig { + tests := make([]TestConfig, len(names)) for i := range names { - tests[i] = &TestConfigImpl{Name: names[i], FromScript: false} + tests[i] = &testConfigImpl{Name: names[i], FromScript: false} } return tests } diff --git a/go/performance/utils/sysbench_runner/sysbench_config_test.go b/go/performance/utils/sysbench_runner/sysbench_config_test.go index 8fa83a954d6..a58c2946c50 100644 --- a/go/performance/utils/sysbench_runner/sysbench_config_test.go +++ b/go/performance/utils/sysbench_runner/sysbench_config_test.go @@ -23,32 +23,32 @@ import ( var testIdFunc = func() string { return "id" } func TestConfigTestGetTests(t *testing.T) { - empty := &TestConfigImpl{Name: "test_name"} + empty := &testConfigImpl{Name: "test_name"} - one := &TestConfigImpl{Name: "test_one", N: 3} - two := &TestConfigImpl{Name: "test_two", N: 2} - three := &TestConfigImpl{Name: "test_three", N: 1} + one := &testConfigImpl{Name: "test_one", N: 3} + two := &testConfigImpl{Name: "test_two", N: 2} + three := &testConfigImpl{Name: "test_three", N: 1} - opts := &TestConfigImpl{ + opts := &testConfigImpl{ Name: "test_options", N: 1, Options: []string{"--create_secondary=on", "--auto_inc=off"}, } - serverConfig := &doltServerConfigImpl{Server: MySql, Version: "test-version", Host: "localhost", ResultsFormat: CsvFormat} + serverConfig := &doltServerConfigImpl{Version: "test-version", Host: "localhost", ResultsFormat: CsvFormat} tests := []struct { description string - config *sysbenchRunnerConfigImpl - expectedTests []*sysbenchTestImpl + config SysbenchConfig + expectedTests []SysbenchTest expectedError error }{ { description: "should error if no test name is defined", config: &sysbenchRunnerConfigImpl{ - Servers: []*doltServerConfigImpl{serverConfig}, - Tests: []*TestConfigImpl{ - {Name: ""}, + Servers: []ServerConfig{serverConfig}, + Tests: []TestConfig{ + &testConfigImpl{Name: ""}, }, }, expectedTests: nil, @@ -57,47 +57,48 @@ func TestConfigTestGetTests(t *testing.T) { { description: "should create single test if N is < 1", config: &sysbenchRunnerConfigImpl{ - Servers: []*doltServerConfigImpl{serverConfig}, - Tests: []*TestConfigImpl{empty}, + Servers: []ServerConfig{serverConfig}, + Tests: []TestConfig{empty}, }, - expectedTests: []*sysbenchTestImpl{ - { + expectedTests: []SysbenchTest{ + &sysbenchTestImpl{ id: testIdFunc(), Name: "test_name", - Params: fromConfigTestParams(empty, serverConfig), + Params: serverConfig.GetTestingParams(empty), }, }, }, { description: "should return a test for each N defined on the TestConfigImpl", config: &sysbenchRunnerConfigImpl{ - Servers: []*doltServerConfigImpl{serverConfig}, - Tests: []*TestConfigImpl{one, two, three}, + Servers: []ServerConfig{serverConfig}, + Tests: []TestConfig{one, two, three}, }, - expectedTests: []*sysbenchTestImpl{ - {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, - {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, - {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, - {id: testIdFunc(), Name: "test_two", Params: fromConfigTestParams(two, serverConfig)}, - {id: testIdFunc(), Name: "test_two", Params: fromConfigTestParams(two, serverConfig)}, - {id: testIdFunc(), Name: "test_three", Params: fromConfigTestParams(three, serverConfig)}, + expectedTests: []SysbenchTest{ + &sysbenchTestImpl{id: testIdFunc(), Name: "test_one", Params: serverConfig.GetTestingParams(one)}, + &sysbenchTestImpl{id: testIdFunc(), Name: "test_one", Params: serverConfig.GetTestingParams(one)}, + &sysbenchTestImpl{id: testIdFunc(), Name: "test_one", Params: serverConfig.GetTestingParams(one)}, + &sysbenchTestImpl{id: testIdFunc(), Name: "test_two", Params: serverConfig.GetTestingParams(two)}, + &sysbenchTestImpl{id: testIdFunc(), Name: "test_two", Params: serverConfig.GetTestingParams(two)}, + &sysbenchTestImpl{id: testIdFunc(), Name: "test_three", Params: serverConfig.GetTestingParams(three)}, }, }, { description: "should apply user options to test params", config: &sysbenchRunnerConfigImpl{ - Servers: []*doltServerConfigImpl{serverConfig}, - Tests: []*TestConfigImpl{opts}, + Servers: []ServerConfig{serverConfig}, + Tests: []TestConfig{opts}, }, - expectedTests: []*sysbenchTestImpl{ - {id: testIdFunc(), Name: "test_options", Params: fromConfigTestParams(opts, serverConfig)}, + expectedTests: []SysbenchTest{ + &sysbenchTestImpl{id: testIdFunc(), Name: "test_options", Params: serverConfig.GetTestingParams(opts)}, }, }, } for _, test := range tests { t.Run(test.description, func(t *testing.T) { - for _, s := range test.config.Servers { - actual, err := GetTests(test.config, s, testIdFunc) + svs := test.config.GetServerConfigs() + for _, s := range svs { + actual, err := GetTests(test.config, s) assert.Equal(t, test.expectedError, err) assert.Equal(t, len(test.expectedTests), len(actual)) assert.ElementsMatch(t, test.expectedTests, actual) diff --git a/go/performance/utils/sysbench_runner/sysbench_tests.go b/go/performance/utils/sysbench_runner/sysbench_tests.go index 723b593aa96..ab1b7c2f94b 100644 --- a/go/performance/utils/sysbench_runner/sysbench_tests.go +++ b/go/performance/utils/sysbench_runner/sysbench_tests.go @@ -32,7 +32,7 @@ type sysbenchTestImpl struct { FromScript bool } -var _ Test = &sysbenchTestImpl{} +var _ SysbenchTest = &sysbenchTestImpl{} func NewSysbenchTest(id, name string, params TestParams, fromScript bool) *sysbenchTestImpl { return &sysbenchTestImpl{ @@ -55,6 +55,10 @@ func (t *sysbenchTestImpl) GetParamsToSlice() []string { return t.Params.ToSlice() } +func (t *sysbenchTestImpl) GetFromScript() bool { + return t.FromScript +} + // PrepareArgs returns a test's args for sysbench's prepare step func (t *sysbenchTestImpl) GetPrepareArgs(serverConfig ServerConfig) []string { return withCommand(t.Params, sysbenchPrepareCommand) From 8049429847b93ff4491289d0d11d98ae033ccde0 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Tue, 13 Feb 2024 15:10:48 -0800 Subject: [PATCH 19/25] /go/performance/utils/sysbench_runner/results_test.go: get tests passing again --- go/performance/utils/sysbench_runner/results_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/go/performance/utils/sysbench_runner/results_test.go b/go/performance/utils/sysbench_runner/results_test.go index 795c2333beb..072cdd831ac 100644 --- a/go/performance/utils/sysbench_runner/results_test.go +++ b/go/performance/utils/sysbench_runner/results_test.go @@ -17,6 +17,7 @@ package sysbench_runner import ( "fmt" "math/rand" + "strings" "testing" "time" @@ -266,6 +267,9 @@ func TestFromOutputResults(t *testing.T) { params := NewSysbenchTestParams() params.Append(testTestParams) + serverParams := defaultDoltServerParams + serverParams = append(serverParams, testServerParams) + tests := []struct { description string output []byte @@ -283,7 +287,6 @@ func TestFromOutputResults(t *testing.T) { RuntimeGoArch: testGoArch, }, serverConfig: &doltServerConfigImpl{ - Host: "localhost", Version: testServerVersion, ServerExec: "test-exec", }, @@ -296,7 +299,8 @@ func TestFromOutputResults(t *testing.T) { SuiteId: testSuiteId, RuntimeOS: testOS, RuntimeGoArch: testGoArch, - ServerName: testServer, + ServerName: string(Dolt), + ServerParams: strings.Join(defaultDoltServerParams, " "), ServerVersion: testServerVersion, TestName: testTestName, TestParams: testTestParams, @@ -317,6 +321,7 @@ func TestFromOutputResults(t *testing.T) { }, }, } + for _, test := range tests { t.Run(test.description, func(t *testing.T) { serverParams, err := test.serverConfig.GetServerArgs() From b368470ab11192916ed2300cba037109ea91051a Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Tue, 13 Feb 2024 16:16:39 -0800 Subject: [PATCH 20/25] Fixing issue with IndexedDoltTable not fully implementing the interface of DoltTable (inconsistent with WritableIndexedDoltTable) --- .../doltcore/sqle/enginetest/dolt_queries.go | 45 +++++++++++++++++++ .../doltcore/sqle/indexed_dolt_table.go | 42 +++-------------- 2 files changed, 52 insertions(+), 35 deletions(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index 512f2234a2f..2fad458b7f7 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -861,6 +861,51 @@ var DoltScripts = []queries.ScriptTest{ }, }, }, + { + Name: "test AS OF indexed queries (https://github.com/dolthub/dolt/issues/7488)", + SetUpScript: []string{ + "create table indexedTable (pk int primary key, c0 int, c1 varchar(255), key c1_idx(c1));", + "insert into indexedTable (pk, c1) values (1, 'one');", + "call dolt_commit('-Am', 'adding table t with index');", + "SET @commit1 = hashof('HEAD');", + + "update indexedTable set c1='two';", + "call dolt_commit('-am', 'updating one to two');", + "SET @commit2 = hashof('HEAD');", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "SELECT c1 from indexedTable;", + Expected: []sql.Row{{"two"}}, + ExpectedIndexes: []string{}, + }, + { + Query: "SELECT c1 from indexedTable where c1 > 'o';", + Expected: []sql.Row{{"two"}}, + ExpectedIndexes: []string{"c1_idx"}, + }, + { + Query: "SELECT c1 from indexedTable as of @commit2;", + Expected: []sql.Row{{"two"}}, + ExpectedIndexes: []string{}, + }, + { + Query: "SELECT c1 from indexedTable as of @commit2 where c1 > 'o';", + Expected: []sql.Row{{"two"}}, + ExpectedIndexes: []string{"c1_idx"}, + }, + { + Query: "SELECT c1 from indexedTable as of @commit1;", + Expected: []sql.Row{{"one"}}, + ExpectedIndexes: []string{}, + }, + { + Query: "SELECT c1 from indexedTable as of @commit1 where c1 > 'o';", + Expected: []sql.Row{{"one"}}, + ExpectedIndexes: []string{"c1_idx"}, + }, + }, + }, { Name: "test as of indexed join (https://github.com/dolthub/dolt/issues/2189)", SetUpScript: []string{ diff --git a/go/libraries/doltcore/sqle/indexed_dolt_table.go b/go/libraries/doltcore/sqle/indexed_dolt_table.go index 941450ff273..910fa3da540 100644 --- a/go/libraries/doltcore/sqle/indexed_dolt_table.go +++ b/go/libraries/doltcore/sqle/indexed_dolt_table.go @@ -27,7 +27,7 @@ import ( // DoltTable, but its RowIter function returns values that match a sql.Range, instead of all // rows. It's returned by the DoltTable.IndexedAccess function. type IndexedDoltTable struct { - table *DoltTable + *DoltTable idx index.DoltIndex lb index.LookupBuilder isDoltFormat bool @@ -36,7 +36,7 @@ type IndexedDoltTable struct { func NewIndexedDoltTable(t *DoltTable, idx index.DoltIndex) *IndexedDoltTable { return &IndexedDoltTable{ - table: t, + DoltTable: t, idx: idx, isDoltFormat: types.IsFormat_DOLT(t.Format()), mu: &sync.Mutex{}, @@ -46,32 +46,8 @@ func NewIndexedDoltTable(t *DoltTable, idx index.DoltIndex) *IndexedDoltTable { var _ sql.IndexedTable = (*IndexedDoltTable)(nil) var _ sql.CommentedTable = (*IndexedDoltTable)(nil) -func (idt *IndexedDoltTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) { - return idt.table.GetIndexes(ctx) -} - -func (idt *IndexedDoltTable) Name() string { - return idt.table.Name() -} - -func (idt *IndexedDoltTable) String() string { - return idt.table.String() -} - -func (idt *IndexedDoltTable) Schema() sql.Schema { - return idt.table.Schema() -} - -func (idt *IndexedDoltTable) Collation() sql.CollationID { - return sql.CollationID(idt.table.sch.GetCollation()) -} - -func (idt *IndexedDoltTable) Comment() string { - return idt.table.Comment() -} - func (idt *IndexedDoltTable) LookupPartitions(ctx *sql.Context, lookup sql.IndexLookup) (sql.PartitionIter, error) { - return index.NewRangePartitionIter(ctx, idt.table, lookup, idt.isDoltFormat) + return index.NewRangePartitionIter(ctx, idt.DoltTable, lookup, idt.isDoltFormat) } func (idt *IndexedDoltTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) { @@ -81,13 +57,13 @@ func (idt *IndexedDoltTable) Partitions(ctx *sql.Context) (sql.PartitionIter, er func (idt *IndexedDoltTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) { idt.mu.Lock() defer idt.mu.Unlock() - key, canCache, err := idt.table.DataCacheKey(ctx) + key, canCache, err := idt.DoltTable.DataCacheKey(ctx) if err != nil { return nil, err } if idt.lb == nil || !canCache || idt.lb.Key() != key { - idt.lb, err = index.NewLookupBuilder(ctx, idt.table, idt.idx, key, idt.table.projectedCols, idt.table.sqlSch, idt.isDoltFormat) + idt.lb, err = index.NewLookupBuilder(ctx, idt.DoltTable, idt.idx, key, idt.DoltTable.projectedCols, idt.DoltTable.sqlSch, idt.isDoltFormat) if err != nil { return nil, err } @@ -99,12 +75,12 @@ func (idt *IndexedDoltTable) PartitionRows(ctx *sql.Context, part sql.Partition) func (idt *IndexedDoltTable) PartitionRows2(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) { idt.mu.Lock() defer idt.mu.Unlock() - key, canCache, err := idt.table.DataCacheKey(ctx) + key, canCache, err := idt.DoltTable.DataCacheKey(ctx) if err != nil { return nil, err } if idt.lb == nil || !canCache || idt.lb.Key() != key { - idt.lb, err = index.NewLookupBuilder(ctx, idt.table, idt.idx, key, idt.table.projectedCols, idt.table.sqlSch, idt.isDoltFormat) + idt.lb, err = index.NewLookupBuilder(ctx, idt.DoltTable, idt.idx, key, idt.DoltTable.projectedCols, idt.DoltTable.sqlSch, idt.isDoltFormat) if err != nil { return nil, err } @@ -113,10 +89,6 @@ func (idt *IndexedDoltTable) PartitionRows2(ctx *sql.Context, part sql.Partition return idt.lb.NewRowIter(ctx, part) } -func (idt *IndexedDoltTable) IsTemporary() bool { - return idt.table.IsTemporary() -} - var _ sql.IndexedTable = (*WritableIndexedDoltTable)(nil) var _ sql.UpdatableTable = (*WritableIndexedDoltTable)(nil) var _ sql.DeletableTable = (*WritableIndexedDoltTable)(nil) From aabdce4be997d3c0dbd21cc1a3f934dd786a5658 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Tue, 13 Feb 2024 16:42:49 -0800 Subject: [PATCH 21/25] /go/performance/utils/sysbench_runner: wip, tests working, runner and profile working, need tpcc now, then unmarshal stuff --- .../sysbench_runner/mysql_server_config.go | 10 +- .../utils/sysbench_runner/run_test.go | 148 ++++++++++++++++++ .../utils/sysbench_runner/tpcc_config.go | 4 +- 3 files changed, 159 insertions(+), 3 deletions(-) diff --git a/go/performance/utils/sysbench_runner/mysql_server_config.go b/go/performance/utils/sysbench_runner/mysql_server_config.go index b5714a35d16..8fc9e4126a9 100644 --- a/go/performance/utils/sysbench_runner/mysql_server_config.go +++ b/go/performance/utils/sysbench_runner/mysql_server_config.go @@ -40,7 +40,7 @@ type mysqlServerConfigImpl struct { Socket string } -var _ ServerConfig = &mysqlServerConfigImpl{} +var _ ProtocolServerConfig = &mysqlServerConfigImpl{} func NewMysqlServerConfig(version, serverExec, serverUser, host, resultsFormat, protocol, socket string, port int, serverArgs []string, skipBinLog bool) *mysqlServerConfigImpl { return &mysqlServerConfigImpl{ @@ -86,6 +86,14 @@ func (sc *mysqlServerConfigImpl) GetResultsFormat() string { return sc.ResultsFormat } +func (sc *mysqlServerConfigImpl) GetConnectionProtocol() string { + return sc.ConnectionProtocol +} + +func (sc *mysqlServerConfigImpl) GetSocket() string { + return sc.Socket +} + func (sc *mysqlServerConfigImpl) GetServerArgs() ([]string, error) { params := make([]string, 0) if sc.ServerUser != "" { diff --git a/go/performance/utils/sysbench_runner/run_test.go b/go/performance/utils/sysbench_runner/run_test.go index ffc24a61881..f8e28e2aa36 100644 --- a/go/performance/utils/sysbench_runner/run_test.go +++ b/go/performance/utils/sysbench_runner/run_test.go @@ -16,8 +16,11 @@ package sysbench_runner import ( "context" + "errors" + "fmt" "log" "os" + "path/filepath" "testing" ) @@ -65,3 +68,148 @@ func selectTests(names ...string) []TestConfig { } return tests } + +func TestDoltMysqlSysbenchRunner(t *testing.T) { + t.Skip() + dir := t.TempDir() + log.Println(dir) + err := os.Chdir(dir) + if err != nil { + log.Fatal(err) + } + + conf := &sysbenchRunnerConfigImpl{ + Tests: []TestConfig{ + NewTestConfig("oltp_read_write", nil, false), + NewTestConfig("oltp_update_index", nil, false), + NewTestConfig("oltp_delete_insert", nil, true), + }, + Servers: []ServerConfig{ + &doltServerConfigImpl{ + Id: "test-dolt", + Version: "1.33.0", + ResultsFormat: CsvFormat, + ServerExec: "/Users/dustin/go/bin/dolt", + }, + &mysqlServerConfigImpl{ + Id: "test-mysql", + Host: "127.0.0.1", + Port: 3606, + Version: "8.0.35", + ResultsFormat: CsvFormat, + ServerExec: "/opt/homebrew/bin/mysqld", + ServerUser: "root", + SkipLogBin: true, + ConnectionProtocol: "tcp", + }, + }, + ScriptDir: "/Users/dustin/src/sysbench-lua-scripts", + TestOptions: []string{ + "--rand-seed=1", + "--table-size=10000", + "--rand-type=uniform", + "--time=120", + "--percentile=50", + }, + InitBigRepo: true, + } + + err = Run(context.Background(), conf) + if err != nil { + log.Fatal(err) + } +} + +func TestDoltgresPostgresSysbenchRunner(t *testing.T) { + t.Skip() + dir := t.TempDir() + log.Println(dir) + err := os.Chdir(dir) + if err != nil { + log.Fatal(err) + } + + conf := &sysbenchRunnerConfigImpl{ + Tests: []TestConfig{ + NewTestConfig("oltp_read_write", nil, false), + NewTestConfig("oltp_update_index", nil, false), + }, + Servers: []ServerConfig{ + &postgresServerConfigImpl{ + Id: "test-postgres", + Host: "127.0.0.1", + Version: "15.5", + ResultsFormat: CsvFormat, + ServerExec: "/opt/homebrew/opt/postgresql@15/bin/postgres", + InitExec: "/opt/homebrew/opt/postgresql@15/bin/initdb", + ServerUser: "root", + }, + &doltgresServerConfigImpl{ + Id: "test-doltgres", + Port: 4433, + Host: "127.0.0.1", + Version: "b139dfb", + ResultsFormat: CsvFormat, + ServerExec: "/Users/dustin/go/bin/doltgres", + }, + }, + ScriptDir: "/Users/dustin/src/sysbench-lua-scripts", + TestOptions: []string{ + "--rand-seed=1", + "--table-size=10000", + "--rand-type=uniform", + "--time=120", + "--percentile=50", + }, + InitBigRepo: true, + } + + err = Run(context.Background(), conf) + if err != nil { + log.Fatal(err) + } +} + +func TestDoltProfiler(t *testing.T) { + t.Skip() + dir := t.TempDir() + log.Println(dir) + err := os.Chdir(dir) + if err != nil { + log.Fatal(err) + } + + id := "test-dolt-profile" + conf := &sysbenchRunnerConfigImpl{ + Tests: []TestConfig{ + NewTestConfig("oltp_read_write", nil, false), + }, + Servers: []ServerConfig{ + &doltServerConfigImpl{ + Id: id, + Version: "1.33.0", + ResultsFormat: CsvFormat, + ServerExec: "/Users/dustin/go/bin/dolt", + ServerProfile: CpuServerProfile, + ProfilePath: dir, + }, + }, + TestOptions: []string{ + "--rand-seed=1", + "--table-size=10000", + "--rand-type=uniform", + "--time=30", + "--percentile=50", + }, + } + + err = Run(context.Background(), conf) + if err != nil { + log.Fatal(err) + } + + expectedProfile := filepath.Join(dir, fmt.Sprintf("%s_%s", id, cpuProfileFilename)) + if _, err := os.Stat(expectedProfile); errors.Is(err, os.ErrNotExist) { + log.Fatal("failed to create dolt cpu profile") + } +} diff --git a/go/performance/utils/sysbench_runner/tpcc_config.go b/go/performance/utils/sysbench_runner/tpcc_config.go index 5d7c4445117..c462cbb0d00 100644 --- a/go/performance/utils/sysbench_runner/tpcc_config.go +++ b/go/performance/utils/sysbench_runner/tpcc_config.go @@ -51,7 +51,7 @@ type tpccConfigImpl struct { var _ TpccConfig = &tpccConfigImpl{} -func NewTpccConfig() *tpccConfigImpl { +func NewTpccRunnerConfig() *tpccConfigImpl { return &tpccConfigImpl{ Servers: make([]ServerConfig, 0), ScaleFactors: make([]int, 0), @@ -153,7 +153,7 @@ func FromFileTpccConfig(configPath string) (TpccConfig, error) { return nil, err } - config := NewTpccConfig() + config := NewTpccRunnerConfig() err = json.Unmarshal(data, config) if err != nil { return nil, err From fd8fcd5a0b85f0f53075960a3b86fd3bf7c6f68e Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 13 Feb 2024 12:34:33 -0800 Subject: [PATCH 22/25] Formatter complaining about no-op append use for reasons unknown --- go/cmd/dolt/commands/engine/sqlengine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 5e72e052665..103f082fc69 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -115,7 +115,7 @@ func NewSqlEngine( return nil, err } - all := append(dbs) + all := dbs[:] clusterDB := config.ClusterController.ClusterDatabase() if clusterDB != nil { From ce21c0b610468ed4749cf8580a507f61998c4f17 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Tue, 13 Feb 2024 17:18:33 -0800 Subject: [PATCH 23/25] /go/performance/utils/sysbench_runner: wip, all seems to work, need to fix json unmarshalling --- .../sysbench_runner/dolt_server_config.go | 3 +- .../sysbench_runner/doltgres_server_config.go | 1 + .../sysbench_runner/mysql_server_config.go | 1 + .../sysbench_runner/postgres_server_config.go | 1 + .../utils/sysbench_runner/results.go | 3 +- .../utils/sysbench_runner/run_test.go | 38 +++++++++++++++++++ go/performance/utils/sysbench_runner/tpcc.go | 6 ++- .../utils/sysbench_runner/tpcc_tests.go | 10 ++++- 8 files changed, 59 insertions(+), 4 deletions(-) diff --git a/go/performance/utils/sysbench_runner/dolt_server_config.go b/go/performance/utils/sysbench_runner/dolt_server_config.go index b5358f43938..61840d0d316 100644 --- a/go/performance/utils/sysbench_runner/dolt_server_config.go +++ b/go/performance/utils/sysbench_runner/dolt_server_config.go @@ -2,8 +2,9 @@ package sysbench_runner import ( "fmt" - "github.com/google/uuid" "os" + + "github.com/google/uuid" ) type ServerProfile string diff --git a/go/performance/utils/sysbench_runner/doltgres_server_config.go b/go/performance/utils/sysbench_runner/doltgres_server_config.go index cd9ff3a3fba..e073f1be15e 100644 --- a/go/performance/utils/sysbench_runner/doltgres_server_config.go +++ b/go/performance/utils/sysbench_runner/doltgres_server_config.go @@ -2,6 +2,7 @@ package sysbench_runner import ( "fmt" + "github.com/google/uuid" ) diff --git a/go/performance/utils/sysbench_runner/mysql_server_config.go b/go/performance/utils/sysbench_runner/mysql_server_config.go index 8fc9e4126a9..a5bdf3f5162 100644 --- a/go/performance/utils/sysbench_runner/mysql_server_config.go +++ b/go/performance/utils/sysbench_runner/mysql_server_config.go @@ -2,6 +2,7 @@ package sysbench_runner import ( "fmt" + "github.com/google/uuid" ) diff --git a/go/performance/utils/sysbench_runner/postgres_server_config.go b/go/performance/utils/sysbench_runner/postgres_server_config.go index e1e790f2e3f..ffefa4cf7f6 100644 --- a/go/performance/utils/sysbench_runner/postgres_server_config.go +++ b/go/performance/utils/sysbench_runner/postgres_server_config.go @@ -2,6 +2,7 @@ package sysbench_runner import ( "fmt" + "github.com/google/uuid" ) diff --git a/go/performance/utils/sysbench_runner/results.go b/go/performance/utils/sysbench_runner/results.go index 10ab10d6e52..a371706bda9 100644 --- a/go/performance/utils/sysbench_runner/results.go +++ b/go/performance/utils/sysbench_runner/results.go @@ -16,10 +16,11 @@ package sysbench_runner import ( "errors" - "github.com/google/uuid" "path/filepath" "strconv" "strings" + + "github.com/google/uuid" ) var ( diff --git a/go/performance/utils/sysbench_runner/run_test.go b/go/performance/utils/sysbench_runner/run_test.go index f8e28e2aa36..ba038797da5 100644 --- a/go/performance/utils/sysbench_runner/run_test.go +++ b/go/performance/utils/sysbench_runner/run_test.go @@ -213,3 +213,41 @@ func TestDoltProfiler(t *testing.T) { log.Fatal("failed to create dolt cpu profile") } } + +func TestDoltMysqlTpccRunner(t *testing.T) { + t.Skip() + dir := t.TempDir() + log.Println(dir) + err := os.Chdir(dir) + if err != nil { + log.Fatal(err) + } + + conf := &tpccConfigImpl{ + Servers: []ServerConfig{ + &doltServerConfigImpl{ + Id: "test-dolt-tpcc", + Version: "1.33.0", + ResultsFormat: CsvFormat, + ServerExec: "/Users/dustin/go/bin/dolt", + }, + &mysqlServerConfigImpl{ + Id: "test-mysql-tpcc", + Host: "127.0.0.1", + Port: 3606, + Version: "8.0.35", + ResultsFormat: CsvFormat, + ServerExec: "/opt/homebrew/bin/mysqld", + ServerUser: "root", + SkipLogBin: true, + ConnectionProtocol: "tcp", + }, + }, + ScriptDir: "/Users/dustin/src/sysbench-tpcc", + } + + err = RunTpcc(context.Background(), conf) + if err != nil { + log.Fatal(err) + } +} diff --git a/go/performance/utils/sysbench_runner/tpcc.go b/go/performance/utils/sysbench_runner/tpcc.go index 25753efd5cc..a4cc1c65799 100644 --- a/go/performance/utils/sysbench_runner/tpcc.go +++ b/go/performance/utils/sysbench_runner/tpcc.go @@ -78,7 +78,11 @@ func (t *tpccTesterImpl) cleanup(ctx context.Context) error { args := t.test.GetCleanupArgs(t.serverConfig) cmd := ExecCommand(ctx, t.tpccCommand, args...) cmd = t.updateCmdEnv(cmd) - return cmd.Run() + err := cmd.Run() + if err != nil { + return err + } + return nil } func (t *tpccTesterImpl) Test(ctx context.Context) (*Result, error) { diff --git a/go/performance/utils/sysbench_runner/tpcc_tests.go b/go/performance/utils/sysbench_runner/tpcc_tests.go index f8d7fdda334..20af830152f 100644 --- a/go/performance/utils/sysbench_runner/tpcc_tests.go +++ b/go/performance/utils/sysbench_runner/tpcc_tests.go @@ -2,6 +2,7 @@ package sysbench_runner import ( "fmt" + "github.com/google/uuid" ) @@ -110,6 +111,10 @@ func (t *tpccTestImpl) doltArgs(serverConfig ServerConfig) []string { args := make([]string, 0) args = append(args, defaultTpccParams...) args = append(args, fmt.Sprintf("%s=%s", tpccMysqlHostFlag, serverConfig.GetHost())) + port := serverConfig.GetPort() + if port > 0 { + args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort())) + } args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort())) args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, defaultMysqlUser)) args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime())) @@ -125,14 +130,17 @@ func (t *tpccTestImpl) mysqlArgs(serverConfig ServerConfig) []string { args := make([]string, 0) args = append(args, defaultTpccParams...) host := serverConfig.GetHost() + port := serverConfig.GetPort() args = append(args, fmt.Sprintf("%s=%s", tpccMysqlHostFlag, host)) if host == defaultHost { args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, tpccMysqlUsername)) args = append(args, fmt.Sprintf("%s=%s", tpccMysqlPasswordFlag, tpccPassLocal)) } else { - args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort())) args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, defaultMysqlUser)) } + if port > 0 { + args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort())) + } args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime())) args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads())) args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval())) From d70b0470a74497d2e7ed3f95dff63c0963823877 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Wed, 14 Feb 2024 10:17:31 -0800 Subject: [PATCH 24/25] /go/performance/utils/benchmark_runner: done refactoring --- .../workflows/ci-sysbench-runner-tests.yaml | 48 ++++++------- .../README.md | 13 ---- .../utils/benchmark_runner/benchmark.go | 21 ++++++ .../utils/benchmark_runner/config.go | 39 +++++++++++ .../constants.go | 16 ++++- .../csv.go | 2 +- .../csv_test.go | 2 +- .../debug.go | 2 +- .../dolt.go | 16 ++++- .../dolt_server_config.go | 16 ++++- .../dolt_tpcc.go | 16 ++++- .../doltgres.go | 16 ++++- .../doltgres_server_config.go | 16 ++++- .../json.go | 2 +- .../json_test.go | 2 +- .../mysql.go | 16 ++++- .../mysql_server_config.go | 16 ++++- .../mysql_tpcc.go | 16 ++++- .../postgres.go | 16 ++++- .../postgres_server_config.go | 16 ++++- .../profile.go | 16 ++++- .../results.go | 2 +- .../results_test.go | 2 +- .../run.go | 2 +- .../run_test.go | 2 +- .../run_tpcc.go | 2 +- .../server.go | 16 ++++- .../server_config.go | 16 ++++- .../sysbench.go | 16 ++++- .../sysbench_config.go | 19 +---- .../sysbench_config_test.go | 2 +- .../sysbench_tests.go | 16 ++++- .../test_config.go | 16 ++++- .../tester.go | 16 ++++- .../tpcc.go | 16 ++++- .../tpcc_config.go | 20 +----- .../tpcc_tests.go | 16 ++++- .../utils/sysbench_runner/benchmark.go | 7 -- .../utils/sysbench_runner/cmd/main.go | 70 ------------------- .../utils/sysbench_runner/config.go | 25 ------- 40 files changed, 397 insertions(+), 207 deletions(-) rename go/performance/utils/{sysbench_runner => benchmark_runner}/README.md (92%) create mode 100644 go/performance/utils/benchmark_runner/benchmark.go create mode 100644 go/performance/utils/benchmark_runner/config.go rename go/performance/utils/{sysbench_runner => benchmark_runner}/constants.go (92%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/csv.go (99%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/csv_test.go (98%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/debug.go (97%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/dolt.go (86%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/dolt_server_config.go (86%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/dolt_tpcc.go (80%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/doltgres.go (84%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/doltgres_server_config.go (83%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/json.go (98%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/json_test.go (98%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/mysql.go (85%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/mysql_server_config.go (86%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/mysql_tpcc.go (75%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/postgres.go (83%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/postgres_server_config.go (84%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/profile.go (82%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/results.go (99%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/results_test.go (99%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/run.go (99%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/run_test.go (99%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/run_tpcc.go (98%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/server.go (80%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/server_config.go (50%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/sysbench.go (85%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/sysbench_config.go (96%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/sysbench_config_test.go (99%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/sysbench_tests.go (75%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/test_config.go (71%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/tester.go (53%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/tpcc.go (80%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/tpcc_config.go (89%) rename go/performance/utils/{sysbench_runner => benchmark_runner}/tpcc_tests.go (92%) delete mode 100644 go/performance/utils/sysbench_runner/benchmark.go delete mode 100644 go/performance/utils/sysbench_runner/cmd/main.go delete mode 100644 go/performance/utils/sysbench_runner/config.go diff --git a/.github/workflows/ci-sysbench-runner-tests.yaml b/.github/workflows/ci-sysbench-runner-tests.yaml index ea1ec9e5a1c..0b2535f8641 100644 --- a/.github/workflows/ci-sysbench-runner-tests.yaml +++ b/.github/workflows/ci-sysbench-runner-tests.yaml @@ -1,24 +1,24 @@ -name: Test Sysbench Runner Utility Works - -on: - pull_request: - branches: [ main ] - paths: - - 'go/**' - - 'integration-tests/**' - -concurrency: - group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - mysql_client_integrations_job: - runs-on: ubuntu-22.04 - name: Test Sysbench Runner - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Copy Dockerfile - run: cp -r ./go/performance/continuous_integration/. . - - name: Test sysbench runner - uses: ./.github/actions/sysbench-runner-tests +#name: Test Sysbench Runner Utility Works +# +#on: +# pull_request: +# branches: [ main ] +# paths: +# - 'go/**' +# - 'integration-tests/**' +# +#concurrency: +# group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# mysql_client_integrations_job: +# runs-on: ubuntu-22.04 +# name: Test Sysbench Runner +# steps: +# - name: Checkout +# uses: actions/checkout@v3 +# - name: Copy Dockerfile +# run: cp -r ./go/performance/continuous_integration/. . +# - name: Test sysbench runner +# uses: ./.github/actions/sysbench-runner-tests diff --git a/go/performance/utils/sysbench_runner/README.md b/go/performance/utils/benchmark_runner/README.md similarity index 92% rename from go/performance/utils/sysbench_runner/README.md rename to go/performance/utils/benchmark_runner/README.md index f5a02c9dc8c..5d47db7395c 100644 --- a/go/performance/utils/sysbench_runner/README.md +++ b/go/performance/utils/benchmark_runner/README.md @@ -1,11 +1,6 @@ Sysbench runner is a tool for running sysbench tests against sql servers. Custom sysbench lua scripts used for benchmarking Dolt are [here](https://github.com/dolthub/sysbench-lua-scripts). -The tool requires a json config file to run: -```bash -$ sysbench_runner --config=config.json -``` - Configuration: ```json @@ -62,8 +57,6 @@ oltp_update_non_index `Port` is the server port. Defaults to **3306** for `dolt` and `mysql` Servers. (**Optional**) -`Server` is the server. Only `dolt` and `mysql` are supported. (**Required**) - `Version` is the server version. (**Required**) `ResultsFormat` is the format the results should be written in. Only `json` and `csv` are supported. (**Required**) @@ -105,12 +98,6 @@ Note: Be sure that all mysql processes are off when running this locally. TPCC runner is a tool for running TPCC tests against sql servers. These tests run against the Percona Labs repo [here](https://github.com/Percona-Lab/sysbench-tpcc). -The tool requires a json config file to run. - -```bash -$ go run cmd/main.go --config=sample-tpcc-config.json -``` - Note to this run this locally you need to have the TPCC repo cloned. The `ScriptDir` variable should then be linked to the path of the cloned repo. diff --git a/go/performance/utils/benchmark_runner/benchmark.go b/go/performance/utils/benchmark_runner/benchmark.go new file mode 100644 index 00000000000..a2556180742 --- /dev/null +++ b/go/performance/utils/benchmark_runner/benchmark.go @@ -0,0 +1,21 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import "context" + +type Benchmarker interface { + Benchmark(ctx context.Context) (Results, error) +} diff --git a/go/performance/utils/benchmark_runner/config.go b/go/performance/utils/benchmark_runner/config.go new file mode 100644 index 00000000000..477a5613c08 --- /dev/null +++ b/go/performance/utils/benchmark_runner/config.go @@ -0,0 +1,39 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import "context" + +type Config interface { + GetRuns() int + GetScriptDir() string + GetNomsBinFormat() string + GetRuntimeOs() string + GetRuntimeGoArch() string + GetServerConfigs() []ServerConfig + Validate(ctx context.Context) error + ContainsServerOfType(server ServerType) bool +} + +type SysbenchConfig interface { + Config + GetTestOptions() []string + GetTestConfigs() []TestConfig +} + +type TpccConfig interface { + Config + GetScaleFactors() []int +} diff --git a/go/performance/utils/sysbench_runner/constants.go b/go/performance/utils/benchmark_runner/constants.go similarity index 92% rename from go/performance/utils/sysbench_runner/constants.go rename to go/performance/utils/benchmark_runner/constants.go index 63440d37e26..389882e4be3 100644 --- a/go/performance/utils/sysbench_runner/constants.go +++ b/go/performance/utils/benchmark_runner/constants.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import "time" diff --git a/go/performance/utils/sysbench_runner/csv.go b/go/performance/utils/benchmark_runner/csv.go similarity index 99% rename from go/performance/utils/sysbench_runner/csv.go rename to go/performance/utils/benchmark_runner/csv.go index 44a445fe529..ba40ad66d0c 100644 --- a/go/performance/utils/sysbench_runner/csv.go +++ b/go/performance/utils/benchmark_runner/csv.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "encoding/csv" diff --git a/go/performance/utils/sysbench_runner/csv_test.go b/go/performance/utils/benchmark_runner/csv_test.go similarity index 98% rename from go/performance/utils/sysbench_runner/csv_test.go rename to go/performance/utils/benchmark_runner/csv_test.go index 718c6a1b743..d530028828f 100644 --- a/go/performance/utils/sysbench_runner/csv_test.go +++ b/go/performance/utils/benchmark_runner/csv_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "fmt" diff --git a/go/performance/utils/sysbench_runner/debug.go b/go/performance/utils/benchmark_runner/debug.go similarity index 97% rename from go/performance/utils/sysbench_runner/debug.go rename to go/performance/utils/benchmark_runner/debug.go index d51f2a0ad01..2671d42347f 100644 --- a/go/performance/utils/sysbench_runner/debug.go +++ b/go/performance/utils/benchmark_runner/debug.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/benchmark_runner/dolt.go similarity index 86% rename from go/performance/utils/sysbench_runner/dolt.go rename to go/performance/utils/benchmark_runner/dolt.go index 4aeb4ec9297..ec333a4471a 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/benchmark_runner/dolt.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/dolt_server_config.go b/go/performance/utils/benchmark_runner/dolt_server_config.go similarity index 86% rename from go/performance/utils/sysbench_runner/dolt_server_config.go rename to go/performance/utils/benchmark_runner/dolt_server_config.go index 61840d0d316..6417fe3368a 100644 --- a/go/performance/utils/sysbench_runner/dolt_server_config.go +++ b/go/performance/utils/benchmark_runner/dolt_server_config.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "fmt" diff --git a/go/performance/utils/sysbench_runner/dolt_tpcc.go b/go/performance/utils/benchmark_runner/dolt_tpcc.go similarity index 80% rename from go/performance/utils/sysbench_runner/dolt_tpcc.go rename to go/performance/utils/benchmark_runner/dolt_tpcc.go index 222129bd513..e8b57c9ed95 100644 --- a/go/performance/utils/sysbench_runner/dolt_tpcc.go +++ b/go/performance/utils/benchmark_runner/dolt_tpcc.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/doltgres.go b/go/performance/utils/benchmark_runner/doltgres.go similarity index 84% rename from go/performance/utils/sysbench_runner/doltgres.go rename to go/performance/utils/benchmark_runner/doltgres.go index b2dc6e9e08b..be77c38cece 100644 --- a/go/performance/utils/sysbench_runner/doltgres.go +++ b/go/performance/utils/benchmark_runner/doltgres.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/doltgres_server_config.go b/go/performance/utils/benchmark_runner/doltgres_server_config.go similarity index 83% rename from go/performance/utils/sysbench_runner/doltgres_server_config.go rename to go/performance/utils/benchmark_runner/doltgres_server_config.go index e073f1be15e..caa0af6833e 100644 --- a/go/performance/utils/sysbench_runner/doltgres_server_config.go +++ b/go/performance/utils/benchmark_runner/doltgres_server_config.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "fmt" diff --git a/go/performance/utils/sysbench_runner/json.go b/go/performance/utils/benchmark_runner/json.go similarity index 98% rename from go/performance/utils/sysbench_runner/json.go rename to go/performance/utils/benchmark_runner/json.go index 389acd8c63f..48270d1372f 100644 --- a/go/performance/utils/sysbench_runner/json.go +++ b/go/performance/utils/benchmark_runner/json.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "encoding/json" diff --git a/go/performance/utils/sysbench_runner/json_test.go b/go/performance/utils/benchmark_runner/json_test.go similarity index 98% rename from go/performance/utils/sysbench_runner/json_test.go rename to go/performance/utils/benchmark_runner/json_test.go index c49b1b8d988..b3aaa1dabac 100644 --- a/go/performance/utils/sysbench_runner/json_test.go +++ b/go/performance/utils/benchmark_runner/json_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "fmt" diff --git a/go/performance/utils/sysbench_runner/mysql.go b/go/performance/utils/benchmark_runner/mysql.go similarity index 85% rename from go/performance/utils/sysbench_runner/mysql.go rename to go/performance/utils/benchmark_runner/mysql.go index 1fa732a33ea..2f096838da1 100644 --- a/go/performance/utils/sysbench_runner/mysql.go +++ b/go/performance/utils/benchmark_runner/mysql.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/mysql_server_config.go b/go/performance/utils/benchmark_runner/mysql_server_config.go similarity index 86% rename from go/performance/utils/sysbench_runner/mysql_server_config.go rename to go/performance/utils/benchmark_runner/mysql_server_config.go index a5bdf3f5162..b2cd4f5fa43 100644 --- a/go/performance/utils/sysbench_runner/mysql_server_config.go +++ b/go/performance/utils/benchmark_runner/mysql_server_config.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "fmt" diff --git a/go/performance/utils/sysbench_runner/mysql_tpcc.go b/go/performance/utils/benchmark_runner/mysql_tpcc.go similarity index 75% rename from go/performance/utils/sysbench_runner/mysql_tpcc.go rename to go/performance/utils/benchmark_runner/mysql_tpcc.go index ad88919db4e..93d577e5dcc 100644 --- a/go/performance/utils/sysbench_runner/mysql_tpcc.go +++ b/go/performance/utils/benchmark_runner/mysql_tpcc.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/postgres.go b/go/performance/utils/benchmark_runner/postgres.go similarity index 83% rename from go/performance/utils/sysbench_runner/postgres.go rename to go/performance/utils/benchmark_runner/postgres.go index c976c9cda85..dcbe4af522f 100644 --- a/go/performance/utils/sysbench_runner/postgres.go +++ b/go/performance/utils/benchmark_runner/postgres.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/postgres_server_config.go b/go/performance/utils/benchmark_runner/postgres_server_config.go similarity index 84% rename from go/performance/utils/sysbench_runner/postgres_server_config.go rename to go/performance/utils/benchmark_runner/postgres_server_config.go index ffefa4cf7f6..85d79d89e1f 100644 --- a/go/performance/utils/sysbench_runner/postgres_server_config.go +++ b/go/performance/utils/benchmark_runner/postgres_server_config.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "fmt" diff --git a/go/performance/utils/sysbench_runner/profile.go b/go/performance/utils/benchmark_runner/profile.go similarity index 82% rename from go/performance/utils/sysbench_runner/profile.go rename to go/performance/utils/benchmark_runner/profile.go index 0a86a05ddfd..53da1c06212 100644 --- a/go/performance/utils/sysbench_runner/profile.go +++ b/go/performance/utils/benchmark_runner/profile.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/results.go b/go/performance/utils/benchmark_runner/results.go similarity index 99% rename from go/performance/utils/sysbench_runner/results.go rename to go/performance/utils/benchmark_runner/results.go index a371706bda9..9d7fed6be05 100644 --- a/go/performance/utils/sysbench_runner/results.go +++ b/go/performance/utils/benchmark_runner/results.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "errors" diff --git a/go/performance/utils/sysbench_runner/results_test.go b/go/performance/utils/benchmark_runner/results_test.go similarity index 99% rename from go/performance/utils/sysbench_runner/results_test.go rename to go/performance/utils/benchmark_runner/results_test.go index 072cdd831ac..4ed9a64844c 100644 --- a/go/performance/utils/sysbench_runner/results_test.go +++ b/go/performance/utils/benchmark_runner/results_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "fmt" diff --git a/go/performance/utils/sysbench_runner/run.go b/go/performance/utils/benchmark_runner/run.go similarity index 99% rename from go/performance/utils/sysbench_runner/run.go rename to go/performance/utils/benchmark_runner/run.go index 87426b56e8b..a22964f41d6 100644 --- a/go/performance/utils/sysbench_runner/run.go +++ b/go/performance/utils/benchmark_runner/run.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/run_test.go b/go/performance/utils/benchmark_runner/run_test.go similarity index 99% rename from go/performance/utils/sysbench_runner/run_test.go rename to go/performance/utils/benchmark_runner/run_test.go index ba038797da5..ce66f4ab910 100644 --- a/go/performance/utils/sysbench_runner/run_test.go +++ b/go/performance/utils/benchmark_runner/run_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/run_tpcc.go b/go/performance/utils/benchmark_runner/run_tpcc.go similarity index 98% rename from go/performance/utils/sysbench_runner/run_tpcc.go rename to go/performance/utils/benchmark_runner/run_tpcc.go index 142a6cd4feb..10a3710ca8e 100644 --- a/go/performance/utils/sysbench_runner/run_tpcc.go +++ b/go/performance/utils/benchmark_runner/run_tpcc.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/server.go b/go/performance/utils/benchmark_runner/server.go similarity index 80% rename from go/performance/utils/sysbench_runner/server.go rename to go/performance/utils/benchmark_runner/server.go index ee0aca40977..fbbca1f5c63 100644 --- a/go/performance/utils/sysbench_runner/server.go +++ b/go/performance/utils/benchmark_runner/server.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/server_config.go b/go/performance/utils/benchmark_runner/server_config.go similarity index 50% rename from go/performance/utils/sysbench_runner/server_config.go rename to go/performance/utils/benchmark_runner/server_config.go index a1581ac74cf..37c02c41a77 100644 --- a/go/performance/utils/sysbench_runner/server_config.go +++ b/go/performance/utils/benchmark_runner/server_config.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner type ServerType string diff --git a/go/performance/utils/sysbench_runner/sysbench.go b/go/performance/utils/benchmark_runner/sysbench.go similarity index 85% rename from go/performance/utils/sysbench_runner/sysbench.go rename to go/performance/utils/benchmark_runner/sysbench.go index 69b1eccd3c1..71ff2230531 100644 --- a/go/performance/utils/sysbench_runner/sysbench.go +++ b/go/performance/utils/benchmark_runner/sysbench.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/sysbench_config.go b/go/performance/utils/benchmark_runner/sysbench_config.go similarity index 96% rename from go/performance/utils/sysbench_runner/sysbench_config.go rename to go/performance/utils/benchmark_runner/sysbench_config.go index da5af42bfaf..dee682c8a33 100644 --- a/go/performance/utils/sysbench_runner/sysbench_config.go +++ b/go/performance/utils/benchmark_runner/sysbench_config.go @@ -12,11 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "context" - "encoding/json" "errors" "fmt" "io/fs" @@ -326,22 +325,6 @@ func GetTests(config SysbenchConfig, serverConfig ServerConfig) ([]Test, error) return flattened, nil } -// FromFileConfig returns a validated sysbenchRunnerConfigImpl based on the config file at the configPath -func FromFileConfig(configPath string) (SysbenchConfig, error) { - data, err := os.ReadFile(configPath) - if err != nil { - return nil, err - } - - config := NewRunnerConfig() - err = json.Unmarshal(data, config) - if err != nil { - return nil, err - } - - return config, nil -} - func getMustSupplyError(name string) error { return fmt.Errorf("Must supply %s", name) } diff --git a/go/performance/utils/sysbench_runner/sysbench_config_test.go b/go/performance/utils/benchmark_runner/sysbench_config_test.go similarity index 99% rename from go/performance/utils/sysbench_runner/sysbench_config_test.go rename to go/performance/utils/benchmark_runner/sysbench_config_test.go index a58c2946c50..47856c0e438 100644 --- a/go/performance/utils/sysbench_runner/sysbench_config_test.go +++ b/go/performance/utils/benchmark_runner/sysbench_config_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "testing" diff --git a/go/performance/utils/sysbench_runner/sysbench_tests.go b/go/performance/utils/benchmark_runner/sysbench_tests.go similarity index 75% rename from go/performance/utils/sysbench_runner/sysbench_tests.go rename to go/performance/utils/benchmark_runner/sysbench_tests.go index ab1b7c2f94b..496bfea4caa 100644 --- a/go/performance/utils/sysbench_runner/sysbench_tests.go +++ b/go/performance/utils/benchmark_runner/sysbench_tests.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner type sysbenchTestParamsImpl struct { params []string diff --git a/go/performance/utils/sysbench_runner/test_config.go b/go/performance/utils/benchmark_runner/test_config.go similarity index 71% rename from go/performance/utils/sysbench_runner/test_config.go rename to go/performance/utils/benchmark_runner/test_config.go index 6e23e2acf81..95f3d4aef83 100644 --- a/go/performance/utils/sysbench_runner/test_config.go +++ b/go/performance/utils/benchmark_runner/test_config.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import "github.com/google/uuid" diff --git a/go/performance/utils/sysbench_runner/tester.go b/go/performance/utils/benchmark_runner/tester.go similarity index 53% rename from go/performance/utils/sysbench_runner/tester.go rename to go/performance/utils/benchmark_runner/tester.go index faba21e1937..7a2bed306cc 100644 --- a/go/performance/utils/sysbench_runner/tester.go +++ b/go/performance/utils/benchmark_runner/tester.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import "context" diff --git a/go/performance/utils/sysbench_runner/tpcc.go b/go/performance/utils/benchmark_runner/tpcc.go similarity index 80% rename from go/performance/utils/sysbench_runner/tpcc.go rename to go/performance/utils/benchmark_runner/tpcc.go index a4cc1c65799..e8dc59c33f1 100644 --- a/go/performance/utils/sysbench_runner/tpcc.go +++ b/go/performance/utils/benchmark_runner/tpcc.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "context" diff --git a/go/performance/utils/sysbench_runner/tpcc_config.go b/go/performance/utils/benchmark_runner/tpcc_config.go similarity index 89% rename from go/performance/utils/sysbench_runner/tpcc_config.go rename to go/performance/utils/benchmark_runner/tpcc_config.go index c462cbb0d00..ab568b0177c 100644 --- a/go/performance/utils/sysbench_runner/tpcc_config.go +++ b/go/performance/utils/benchmark_runner/tpcc_config.go @@ -12,13 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "context" - "encoding/json" "fmt" - "os" "runtime" ) @@ -145,19 +143,3 @@ func (c *tpccConfigImpl) Validate(ctx context.Context) error { c.setDefaults() return c.validateServerConfigs() } - -// FromFileTpccConfig returns a validated TpccConfig based on the config file at the configPath -func FromFileTpccConfig(configPath string) (TpccConfig, error) { - data, err := os.ReadFile(configPath) - if err != nil { - return nil, err - } - - config := NewTpccRunnerConfig() - err = json.Unmarshal(data, config) - if err != nil { - return nil, err - } - - return config, nil -} diff --git a/go/performance/utils/sysbench_runner/tpcc_tests.go b/go/performance/utils/benchmark_runner/tpcc_tests.go similarity index 92% rename from go/performance/utils/sysbench_runner/tpcc_tests.go rename to go/performance/utils/benchmark_runner/tpcc_tests.go index 20af830152f..a7df1f10eb9 100644 --- a/go/performance/utils/sysbench_runner/tpcc_tests.go +++ b/go/performance/utils/benchmark_runner/tpcc_tests.go @@ -1,4 +1,18 @@ -package sysbench_runner +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner import ( "fmt" diff --git a/go/performance/utils/sysbench_runner/benchmark.go b/go/performance/utils/sysbench_runner/benchmark.go deleted file mode 100644 index b7c272771a9..00000000000 --- a/go/performance/utils/sysbench_runner/benchmark.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysbench_runner - -import "context" - -type Benchmarker interface { - Benchmark(ctx context.Context) (Results, error) -} diff --git a/go/performance/utils/sysbench_runner/cmd/main.go b/go/performance/utils/sysbench_runner/cmd/main.go deleted file mode 100644 index 07143dc3613..00000000000 --- a/go/performance/utils/sysbench_runner/cmd/main.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "flag" - "log" - "os" - "path/filepath" - - runner "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -var tpcc = flag.Bool("tpcc", false, "run tpcc benchmarks") -var configFile = flag.String("config", "", "path to config file") - -func main() { - flag.Parse() - - if *configFile == "" { - log.Fatal("Must supply config") - } - - configPath, err := filepath.Abs(*configFile) - if err != nil { - log.Fatal(err) - } - if _, err = os.Stat(configPath); os.IsNotExist(err) { - log.Fatal(err) - } - - ctx := context.Background() - if *tpcc { - err = runTpcc(ctx, configPath) - } else { - err = run(ctx, configPath) - } - if err != nil { - log.Fatal(err) - } -} - -func runTpcc(ctx context.Context, configPath string) error { - config, err := runner.FromFileTpccConfig(configPath) - if err != nil { - return err - } - return runner.RunTpcc(ctx, config) -} - -func run(ctx context.Context, configPath string) error { - config, err := runner.FromFileConfig(configPath) - if err != nil { - return err - } - return runner.Run(ctx, config) -} diff --git a/go/performance/utils/sysbench_runner/config.go b/go/performance/utils/sysbench_runner/config.go deleted file mode 100644 index 50ee8a11808..00000000000 --- a/go/performance/utils/sysbench_runner/config.go +++ /dev/null @@ -1,25 +0,0 @@ -package sysbench_runner - -import "context" - -type Config interface { - GetRuns() int - GetScriptDir() string - GetNomsBinFormat() string - GetRuntimeOs() string - GetRuntimeGoArch() string - GetServerConfigs() []ServerConfig - Validate(ctx context.Context) error - ContainsServerOfType(server ServerType) bool -} - -type SysbenchConfig interface { - Config - GetTestOptions() []string - GetTestConfigs() []TestConfig -} - -type TpccConfig interface { - Config - GetScaleFactors() []int -} From f75b598c31c4f24de473579f2df51e5f0d60b48f Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Wed, 14 Feb 2024 10:48:24 -0800 Subject: [PATCH 25/25] /go/performance/utils/benchmark_runner: fix tests --- .../utils/benchmark_runner/run_test.go | 22 +++++-- .../benchmark_runner/sysbench_config_test.go | 61 +++++++++++++------ 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/go/performance/utils/benchmark_runner/run_test.go b/go/performance/utils/benchmark_runner/run_test.go index ce66f4ab910..f97d261e673 100644 --- a/go/performance/utils/benchmark_runner/run_test.go +++ b/go/performance/utils/benchmark_runner/run_test.go @@ -24,8 +24,12 @@ import ( "testing" ) +var runTests = os.Getenv("RUN_BENCHMARK_RUNNER_TESTS") + func TestRunner(t *testing.T) { - t.Skip() + if runTests == "" { + t.Skip() + } dir := t.TempDir() log.Println(dir) err := os.Chdir(dir) @@ -70,7 +74,9 @@ func selectTests(names ...string) []TestConfig { } func TestDoltMysqlSysbenchRunner(t *testing.T) { - t.Skip() + if runTests == "" { + t.Skip() + } dir := t.TempDir() log.Println(dir) err := os.Chdir(dir) @@ -121,7 +127,9 @@ func TestDoltMysqlSysbenchRunner(t *testing.T) { } func TestDoltgresPostgresSysbenchRunner(t *testing.T) { - t.Skip() + if runTests == "" { + t.Skip() + } dir := t.TempDir() log.Println(dir) err := os.Chdir(dir) @@ -171,7 +179,9 @@ func TestDoltgresPostgresSysbenchRunner(t *testing.T) { } func TestDoltProfiler(t *testing.T) { - t.Skip() + if runTests == "" { + t.Skip() + } dir := t.TempDir() log.Println(dir) err := os.Chdir(dir) @@ -215,7 +225,9 @@ func TestDoltProfiler(t *testing.T) { } func TestDoltMysqlTpccRunner(t *testing.T) { - t.Skip() + if runTests == "" { + t.Skip() + } dir := t.TempDir() log.Println(dir) err := os.Chdir(dir) diff --git a/go/performance/utils/benchmark_runner/sysbench_config_test.go b/go/performance/utils/benchmark_runner/sysbench_config_test.go index 47856c0e438..a792a5f41c1 100644 --- a/go/performance/utils/benchmark_runner/sysbench_config_test.go +++ b/go/performance/utils/benchmark_runner/sysbench_config_test.go @@ -20,8 +20,6 @@ import ( "github.com/stretchr/testify/assert" ) -var testIdFunc = func() string { return "id" } - func TestConfigTestGetTests(t *testing.T) { empty := &testConfigImpl{Name: "test_name"} @@ -40,7 +38,7 @@ func TestConfigTestGetTests(t *testing.T) { tests := []struct { description string config SysbenchConfig - expectedTests []SysbenchTest + expectedTests []testTest expectedError error }{ { @@ -60,11 +58,12 @@ func TestConfigTestGetTests(t *testing.T) { Servers: []ServerConfig{serverConfig}, Tests: []TestConfig{empty}, }, - expectedTests: []SysbenchTest{ - &sysbenchTestImpl{ - id: testIdFunc(), - Name: "test_name", - Params: serverConfig.GetTestingParams(empty), + expectedTests: []testTest{ + &testTestImpl{ + &sysbenchTestImpl{ + Name: "test_name", + Params: serverConfig.GetTestingParams(empty), + }, }, }, }, @@ -74,13 +73,13 @@ func TestConfigTestGetTests(t *testing.T) { Servers: []ServerConfig{serverConfig}, Tests: []TestConfig{one, two, three}, }, - expectedTests: []SysbenchTest{ - &sysbenchTestImpl{id: testIdFunc(), Name: "test_one", Params: serverConfig.GetTestingParams(one)}, - &sysbenchTestImpl{id: testIdFunc(), Name: "test_one", Params: serverConfig.GetTestingParams(one)}, - &sysbenchTestImpl{id: testIdFunc(), Name: "test_one", Params: serverConfig.GetTestingParams(one)}, - &sysbenchTestImpl{id: testIdFunc(), Name: "test_two", Params: serverConfig.GetTestingParams(two)}, - &sysbenchTestImpl{id: testIdFunc(), Name: "test_two", Params: serverConfig.GetTestingParams(two)}, - &sysbenchTestImpl{id: testIdFunc(), Name: "test_three", Params: serverConfig.GetTestingParams(three)}, + expectedTests: []testTest{ + &testTestImpl{&sysbenchTestImpl{Name: "test_one", Params: serverConfig.GetTestingParams(one)}}, + &testTestImpl{&sysbenchTestImpl{Name: "test_one", Params: serverConfig.GetTestingParams(one)}}, + &testTestImpl{&sysbenchTestImpl{Name: "test_one", Params: serverConfig.GetTestingParams(one)}}, + &testTestImpl{&sysbenchTestImpl{Name: "test_two", Params: serverConfig.GetTestingParams(two)}}, + &testTestImpl{&sysbenchTestImpl{Name: "test_two", Params: serverConfig.GetTestingParams(two)}}, + &testTestImpl{&sysbenchTestImpl{Name: "test_three", Params: serverConfig.GetTestingParams(three)}}, }, }, { @@ -89,8 +88,8 @@ func TestConfigTestGetTests(t *testing.T) { Servers: []ServerConfig{serverConfig}, Tests: []TestConfig{opts}, }, - expectedTests: []SysbenchTest{ - &sysbenchTestImpl{id: testIdFunc(), Name: "test_options", Params: serverConfig.GetTestingParams(opts)}, + expectedTests: []testTest{ + &testTestImpl{&sysbenchTestImpl{Name: "test_options", Params: serverConfig.GetTestingParams(opts)}}, }, }, } @@ -101,8 +100,34 @@ func TestConfigTestGetTests(t *testing.T) { actual, err := GetTests(test.config, s) assert.Equal(t, test.expectedError, err) assert.Equal(t, len(test.expectedTests), len(actual)) - assert.ElementsMatch(t, test.expectedTests, actual) + updatedExpected := make([]SysbenchTest, len(actual)) + for idx, a := range actual { + e := test.expectedTests[idx] + e.SetId(a.GetId()) + updatedExpected[idx] = e.GetSysbenchTest() + } + assert.ElementsMatch(t, updatedExpected, actual) } }) } } + +type testTest interface { + SetId(id string) + GetSysbenchTest() SysbenchTest + SysbenchTest +} + +type testTestImpl struct { + *sysbenchTestImpl +} + +func (t *testTestImpl) GetSysbenchTest() SysbenchTest { + return t.sysbenchTestImpl +} + +func (t *testTestImpl) SetId(id string) { + t.id = id +} + +var _ testTest = &testTestImpl{}