merge

.github/workflows/ci-sysbench-runner-tests.yaml

@@ -1,24 +1,24 @@
-name: Test Sysbench Runner Utility Works
-
-on:
-  pull_request:
-    branches: [ main ]
-    paths:
-      - 'go/**'
-      - 'integration-tests/**'
-
-concurrency:
-  group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  mysql_client_integrations_job:
-    runs-on: ubuntu-22.04
-    name: Test Sysbench Runner
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Copy Dockerfile
-        run: cp -r ./go/performance/continuous_integration/. .
-      - name: Test sysbench runner
-        uses: ./.github/actions/sysbench-runner-tests
+#name: Test Sysbench Runner Utility Works
+#
+#on:
+#  pull_request:
+#    branches: [ main ]
+#    paths:
+#      - 'go/**'
+#      - 'integration-tests/**'
+#
+#concurrency:
+#  group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }}
+#  cancel-in-progress: true
+#
+#jobs:
+#  mysql_client_integrations_job:
+#    runs-on: ubuntu-22.04
+#    name: Test Sysbench Runner
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v3
+#      - name: Copy Dockerfile
+#        run: cp -r ./go/performance/continuous_integration/. .
+#      - name: Test sysbench runner
+#        uses: ./.github/actions/sysbench-runner-tests

go/cmd/dolt/commands/engine/sqlengine.go

@@ -115,7 +115,7 @@ func NewSqlEngine(
 		return nil, err
 	}
 
-	all := append(dbs)
+	all := dbs[:]
 
 	clusterDB := config.ClusterController.ClusterDatabase()
 	if clusterDB != nil {

go/libraries/doltcore/sqle/cluster/interceptors.go

@@ -204,11 +204,11 @@ type serverinterceptor struct {
 
 func (si *serverinterceptor) Stream() grpc.StreamServerInterceptor {
 	return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
-		fromStandby := false
+		fromClusterMember := false
 		if md, ok := metadata.FromIncomingContext(ss.Context()); ok {
-			fromStandby = si.handleRequestHeaders(md)
+			fromClusterMember = si.handleRequestHeaders(md)
 		}
-		if fromStandby {
+		if fromClusterMember {
 			if err := si.authenticate(ss.Context()); err != nil {
 				return err
 			}
@@ -236,11 +236,11 @@ func (si *serverinterceptor) Stream() grpc.StreamServerInterceptor {
 
 func (si *serverinterceptor) Unary() grpc.UnaryServerInterceptor {
 	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
-		fromStandby := false
+		fromClusterMember := false
 		if md, ok := metadata.FromIncomingContext(ctx); ok {
-			fromStandby = si.handleRequestHeaders(md)
+			fromClusterMember = si.handleRequestHeaders(md)
 		}
-		if fromStandby {
+		if fromClusterMember {
 			if err := si.authenticate(ctx); err != nil {
 				return nil, err
 			}
@@ -271,9 +271,9 @@ func (si *serverinterceptor) handleRequestHeaders(header metadata.MD) bool {
 	epochs := header.Get(clusterRoleEpochHeader)
 	roles := header.Get(clusterRoleHeader)
 	if len(epochs) > 0 && len(roles) > 0 {
-		if roles[0] == string(RolePrimary) && role == RolePrimary {
+		if roles[0] == string(RolePrimary) {
 			if reqepoch, err := strconv.Atoi(epochs[0]); err == nil {
-				if reqepoch == epoch {
+				if reqepoch == epoch && role == RolePrimary {
 					// Misconfiguration in the cluster means this
 					// server and its standby are marked as Primary
 					// at the same epoch. We will become standby
@@ -282,13 +282,17 @@ func (si *serverinterceptor) handleRequestHeaders(header metadata.MD) bool {
 					si.lgr.Errorf("cluster: serverinterceptor: this server and its standby replica are both primary at the same epoch. force transitioning to detected_broken_config.")
 					si.roleSetter(string(RoleDetectedBrokenConfig), reqepoch)
 				} else if reqepoch > epoch {
-					// The client replicating to us thinks it is the primary at a higher epoch than us.
-					si.lgr.Warnf("cluster: serverinterceptor: this server is primary at epoch %d. the server replicating to it is primary at epoch %d. force transitioning to standby.", epoch, reqepoch)
+					if role == RolePrimary {
+						// The client replicating to us thinks it is the primary at a higher epoch than us.
+						si.lgr.Warnf("cluster: serverinterceptor: this server is primary at epoch %d. the server replicating to it is primary at epoch %d. force transitioning to standby.", epoch, reqepoch)
+					} else if role == RoleDetectedBrokenConfig {
+						si.lgr.Warnf("cluster: serverinterceptor: this server is detected_broken_config at epoch %d. the server replicating to it is primary at epoch %d. transitioning to standby.", epoch, reqepoch)
+					}
 					si.roleSetter(string(RoleStandby), reqepoch)
 				}
 			}
 		}
-		// returns true if the request was from a standby replica, false otherwise
+		// returns true if the request was from a cluster replica, false otherwise
 		return true
 	}
 	return false

go/libraries/doltcore/sqle/enginetest/dolt_queries.go

@@ -861,6 +861,51 @@ var DoltScripts = []queries.ScriptTest{
 			},
 		},
 	},
+	{
+		Name: "test AS OF indexed queries (https://github.com/dolthub/dolt/issues/7488)",
+		SetUpScript: []string{
+			"create table indexedTable (pk int primary key, c0 int, c1 varchar(255), key c1_idx(c1));",
+			"insert into indexedTable (pk, c1) values (1, 'one');",
+			"call dolt_commit('-Am', 'adding table t with index');",
+			"SET @commit1 = hashof('HEAD');",
+
+			"update indexedTable set c1='two';",
+			"call dolt_commit('-am', 'updating one to two');",
+			"SET @commit2 = hashof('HEAD');",
+		},
+		Assertions: []queries.ScriptTestAssertion{
+			{
+				Query:           "SELECT c1 from indexedTable;",
+				Expected:        []sql.Row{{"two"}},
+				ExpectedIndexes: []string{},
+			},
+			{
+				Query:           "SELECT c1 from indexedTable where c1 > 'o';",
+				Expected:        []sql.Row{{"two"}},
+				ExpectedIndexes: []string{"c1_idx"},
+			},
+			{
+				Query:           "SELECT c1 from indexedTable as of @commit2;",
+				Expected:        []sql.Row{{"two"}},
+				ExpectedIndexes: []string{},
+			},
+			{
+				Query:           "SELECT c1 from indexedTable as of @commit2 where c1 > 'o';",
+				Expected:        []sql.Row{{"two"}},
+				ExpectedIndexes: []string{"c1_idx"},
+			},
+			{
+				Query:           "SELECT c1 from indexedTable as of @commit1;",
+				Expected:        []sql.Row{{"one"}},
+				ExpectedIndexes: []string{},
+			},
+			{
+				Query:           "SELECT c1 from indexedTable as of @commit1 where c1 > 'o';",
+				Expected:        []sql.Row{{"one"}},
+				ExpectedIndexes: []string{"c1_idx"},
+			},
+		},
+	},
 	{
 		Name: "test as of indexed join (https://github.com/dolthub/dolt/issues/2189)",
 		SetUpScript: []string{

go/libraries/doltcore/sqle/indexed_dolt_table.go

@@ -27,7 +27,7 @@ import (
 // DoltTable, but its RowIter function returns values that match a sql.Range, instead of all
 // rows. It's returned by the DoltTable.IndexedAccess function.
 type IndexedDoltTable struct {
-	table        *DoltTable
+	*DoltTable
 	idx          index.DoltIndex
 	lb           index.LookupBuilder
 	isDoltFormat bool
@@ -36,7 +36,7 @@ type IndexedDoltTable struct {
 
 func NewIndexedDoltTable(t *DoltTable, idx index.DoltIndex) *IndexedDoltTable {
 	return &IndexedDoltTable{
-		table:        t,
+		DoltTable:    t,
 		idx:          idx,
 		isDoltFormat: types.IsFormat_DOLT(t.Format()),
 		mu:           &sync.Mutex{},
@@ -46,32 +46,8 @@ func NewIndexedDoltTable(t *DoltTable, idx index.DoltIndex) *IndexedDoltTable {
 var _ sql.IndexedTable = (*IndexedDoltTable)(nil)
 var _ sql.CommentedTable = (*IndexedDoltTable)(nil)
 
-func (idt *IndexedDoltTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) {
-	return idt.table.GetIndexes(ctx)
-}
-
-func (idt *IndexedDoltTable) Name() string {
-	return idt.table.Name()
-}
-
-func (idt *IndexedDoltTable) String() string {
-	return idt.table.String()
-}
-
-func (idt *IndexedDoltTable) Schema() sql.Schema {
-	return idt.table.Schema()
-}
-
-func (idt *IndexedDoltTable) Collation() sql.CollationID {
-	return sql.CollationID(idt.table.sch.GetCollation())
-}
-
-func (idt *IndexedDoltTable) Comment() string {
-	return idt.table.Comment()
-}
-
 func (idt *IndexedDoltTable) LookupPartitions(ctx *sql.Context, lookup sql.IndexLookup) (sql.PartitionIter, error) {
-	return index.NewRangePartitionIter(ctx, idt.table, lookup, idt.isDoltFormat)
+	return index.NewRangePartitionIter(ctx, idt.DoltTable, lookup, idt.isDoltFormat)
 }
 
 func (idt *IndexedDoltTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) {
@@ -81,13 +57,13 @@ func (idt *IndexedDoltTable) Partitions(ctx *sql.Context) (sql.PartitionIter, er
 func (idt *IndexedDoltTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) {
 	idt.mu.Lock()
 	defer idt.mu.Unlock()
-	key, canCache, err := idt.table.DataCacheKey(ctx)
+	key, canCache, err := idt.DoltTable.DataCacheKey(ctx)
 	if err != nil {
 		return nil, err
 	}
 
 	if idt.lb == nil || !canCache || idt.lb.Key() != key {
-		idt.lb, err = index.NewLookupBuilder(ctx, idt.table, idt.idx, key, idt.table.projectedCols, idt.table.sqlSch, idt.isDoltFormat)
+		idt.lb, err = index.NewLookupBuilder(ctx, idt.DoltTable, idt.idx, key, idt.DoltTable.projectedCols, idt.DoltTable.sqlSch, idt.isDoltFormat)
 		if err != nil {
 			return nil, err
 		}
@@ -99,12 +75,12 @@ func (idt *IndexedDoltTable) PartitionRows(ctx *sql.Context, part sql.Partition)
 func (idt *IndexedDoltTable) PartitionRows2(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) {
 	idt.mu.Lock()
 	defer idt.mu.Unlock()
-	key, canCache, err := idt.table.DataCacheKey(ctx)
+	key, canCache, err := idt.DoltTable.DataCacheKey(ctx)
 	if err != nil {
 		return nil, err
 	}
 	if idt.lb == nil || !canCache || idt.lb.Key() != key {
-		idt.lb, err = index.NewLookupBuilder(ctx, idt.table, idt.idx, key, idt.table.projectedCols, idt.table.sqlSch, idt.isDoltFormat)
+		idt.lb, err = index.NewLookupBuilder(ctx, idt.DoltTable, idt.idx, key, idt.DoltTable.projectedCols, idt.DoltTable.sqlSch, idt.isDoltFormat)
 		if err != nil {
 			return nil, err
 		}
@@ -113,10 +89,6 @@ func (idt *IndexedDoltTable) PartitionRows2(ctx *sql.Context, part sql.Partition
 	return idt.lb.NewRowIter(ctx, part)
 }
 
-func (idt *IndexedDoltTable) IsTemporary() bool {
-	return idt.table.IsTemporary()
-}
-
 var _ sql.IndexedTable = (*WritableIndexedDoltTable)(nil)
 var _ sql.UpdatableTable = (*WritableIndexedDoltTable)(nil)
 var _ sql.DeletableTable = (*WritableIndexedDoltTable)(nil)

go/performance/utils/sysbench_runner/README.md → go/performance/utils/benchmark_runner/README.md

@@ -1,11 +1,6 @@
 Sysbench runner is a tool for running sysbench tests against sql servers. Custom sysbench lua scripts used
 for benchmarking Dolt are [here](https://github.com/dolthub/sysbench-lua-scripts).
 
-The tool requires a json config file to run:
-```bash
-$ sysbench_runner --config=config.json
-```
-
 Configuration:
 
 ```json
@@ -62,8 +57,6 @@ oltp_update_non_index
 
 `Port` is the server port. Defaults to **3306** for `dolt` and `mysql` Servers. (**Optional**)
 
-`Server` is the server. Only `dolt` and `mysql` are supported. (**Required**)
-
 `Version` is the server version. (**Required**)
 
 `ResultsFormat` is the format the results should be written in. Only `json` and `csv` are supported. (**Required**)
@@ -99,3 +92,33 @@ oltp_update_non_index
 `sysbench [options]... [testname] [command]`
 
 Note: Be sure that all mysql processes are off when running this locally.
+
+# TPCC
+
+TPCC runner is a tool for running TPCC tests against sql servers. These tests run against the
+Percona Labs repo [here](https://github.com/Percona-Lab/sysbench-tpcc).
+
+Note to this run this locally you need to have the TPCC repo cloned. The `ScriptDir` variable should then be linked
+to the path of the cloned repo.
+
+Configuration:
+
+```json
+{
+  "Servers": "[...]",
+  "ScriptDir":"/Users/vinairachakonda/go/src/dolthub/sysbench-tpcc",
+  "ScaleFactors": [1]
+}
+```
+
+`Servers`: The server defintions to run the benchmark against. Accepts Dolt and MySQL configuratiosn.
+
+`ScriptDir`: The directory of the TPCC testing scripts
+
+`ScaleFactors`: The number of warehouse to be generated in the test case.
+
+`NomsBinFormat`: The NomsBinFormat to use for this benchmark.
+
+Note that this configuration is still incomplete for the amount of the variable TPCC varies. This intentional as we
+want expose small amounts of independent variables until Dolt gets more robust. See `config.go` to get a breakdown of all the
+variables TPCC varies.

go/performance/utils/benchmark_runner/benchmark.go

@@ -0,0 +1,21 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import "context"
+
+type Benchmarker interface {
+	Benchmark(ctx context.Context) (Results, error)
+}

go/performance/utils/benchmark_runner/config.go

@@ -0,0 +1,39 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import "context"
+
+type Config interface {
+	GetRuns() int
+	GetScriptDir() string
+	GetNomsBinFormat() string
+	GetRuntimeOs() string
+	GetRuntimeGoArch() string
+	GetServerConfigs() []ServerConfig
+	Validate(ctx context.Context) error
+	ContainsServerOfType(server ServerType) bool
+}
+
+type SysbenchConfig interface {
+	Config
+	GetTestOptions() []string
+	GetTestConfigs() []TestConfig
+}
+
+type TpccConfig interface {
+	Config
+	GetScaleFactors() []int
+}