diff --git a/pkg/store/comp/compare_info.go b/pkg/store/comp/compare_info.go new file mode 100644 index 00000000..90d83d8b --- /dev/null +++ b/pkg/store/comp/compare_info.go @@ -0,0 +1,123 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package comp + +import ( + "fmt" + "strings" +) + +// Info contains information about responses difference. +type Info []string + +func (d *Info) Len() int { + return len(*d) +} + +func (d *Info) Add(in ...string) { + *d = append(*d, in...) +} + +func (d *Info) String() string { + return strings.Join(*d, "\n") +} + +func GetCompareInfoSimple(d Results) Info { + lenTest := d.LenRowsTest() + lenOracle := d.LenRowsOracle() + switch { + case lenTest == 0 && lenOracle == 0: + return nil + // responses don`t have rows + case lenTest == lenOracle: + // responses have rows and have same rows count + equalRowsCount := equalRowsSameLen(d) + // equalRowsSameLen function simultaneously deletes equal rows in Test and Oracle stores. + // So we can check rows only one of the stores. + if d.LenRowsTest() < 1 { + return nil + } + return Info{fmt.Sprintf("responses have %d equal rows and unequal rows %d", equalRowsCount, d.LenRowsTest())} + default: + // responses have different rows count + return Info{fmt.Sprintf("different rows count in responses: from test store-%d, from oracle store-%d", lenTest, lenOracle)} + } +} + +func GetCompareInfoDetailed(d Results) Info { + lenTest := d.LenRowsTest() + lenOracle := d.LenRowsOracle() + switch { + case lenTest == 0 && lenOracle == 0: + return nil + // responses don`t have rows + case lenTest < 1 || lenOracle < 1: + // one of the responses without rows. + diff := make(Info, 0) + diff.Add(fmt.Sprintf("different rows count in responses: from test store-%d, from oracle store-%d", lenTest, lenOracle)) + diff.Add(d.StringAllRows("unequal")...) + return diff + case lenTest == lenOracle: + // responses have rows and have same rows count + equalRowsCount := equalRowsSameLen(d) + // equalRowsSameLen function simultaneously deletes equal rows in Test and Oracle stores. + // So we can check rows only one of the stores. + if d.LenRowsTest() < 1 { + return nil + } + diff := make(Info, 0) + diff.Add(fmt.Sprintf("responses have %d equal rows and unequal rows: test store %d; oracle store %d", equalRowsCount, d.LenRowsTest(), d.LenRowsOracle())) + diff.Add(d.StringAllRows("unequal")...) + return diff + default: + // responses have rows and have different rows count + diff := make(Info, 0) + equalRowsCount := equalRowsDiffLen(d) + diff.Add(fmt.Sprintf("responses have %d equal rows and unequal rows: test store %d; oracle store %d", equalRowsCount, d.LenRowsTest(), d.LenRowsOracle())) + diff.Add(d.StringAllRows("unequal")...) + return diff + } +} + +// equalRowsSameLen returns count of equal rows of stores simultaneously deletes equal rows. +// Applies when oracle and test stores have same rows count. +func equalRowsSameLen(d Results) int { + if d.LenRowsTest() == 1 { + return d.EqualSingeRow() + } + equalRowsCount := d.EasyEqualRowsTest() + if d.LenRowsTest() != 0 { + equalRowsCount += d.EqualRowsTest() + } + return equalRowsCount +} + +// equalRowsDiffLen returns count of equal rows of stores simultaneously deletes equal rows. +// Applies when oracle and test stores have different rows count. +func equalRowsDiffLen(d Results) int { + equalRowsCount := 0 + if d.LenRowsTest() > d.LenRowsOracle() { + equalRowsCount = d.EasyEqualRowsOracle() + if d.LenRowsOracle() > 0 { + equalRowsCount += d.EqualRowsOracle() + } + } else { + equalRowsCount = d.EasyEqualRowsTest() + if d.LenRowsTest() > 0 { + equalRowsCount += d.EqualRowsTest() + } + } + return equalRowsCount +} diff --git a/pkg/store/comp/compare_info_mv_test.go b/pkg/store/comp/compare_info_mv_test.go new file mode 100644 index 00000000..bf62e8a9 --- /dev/null +++ b/pkg/store/comp/compare_info_mv_test.go @@ -0,0 +1,137 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package comp + +import ( + "fmt" + "testing" + + "github.com/scylladb/gemini/pkg/store/mv" + "github.com/scylladb/gemini/pkg/store/sv" +) + +func TestGetCompareInfoMV(t *testing.T) { + var test mv.Results + for idx := range testCases { + tests := 2 + if testCases[idx].diffs > 0 { + tests = 4 + } + for tests > 0 { + test.Test.Rows, test.Oracle.Rows, test.Test.Types, test.Oracle.Types = rndSameRowsMV(testCases[idx].test, testCases[idx].oracle) + // Add names + if len(test.Test.Rows) > 0 { + test.Test.Names = make([]string, len(test.Test.Rows[0])) + for col := range test.Test.Rows[0] { + test.Test.Names[col] = fmt.Sprintf("col%d", col) + } + } + if len(test.Oracle.Rows) > 0 { + test.Oracle.Names = make([]string, len(test.Oracle.Rows[0])) + for col := range test.Oracle.Rows[0] { + test.Oracle.Names[col] = fmt.Sprintf("col%d", col) + } + } + if testCases[idx].diffs > 0 { + if tests%2 == 0 { + corruptRows(&test.Test.Rows, testCases[idx].diffs) + } else { + corruptRows(&test.Oracle.Rows, testCases[idx].diffs) + } + } + result := Info{} + errFuncName := "GetCompareInfoDetailed" + if tests%2 == 0 { + errFuncName = "GetCompareInfoSimple" + result = GetCompareInfoSimple(&test) + } else { + result = GetCompareInfoDetailed(&test) + } + if len(result) > 0 && !testCases[idx].haveDif { + t.Fatalf("wrong %s work. test case:%+v \nresult should be empty, but have:%s\n"+ + "mv.Results.Test.Rows:%+v\n"+ + "mv.Results.Oracle.Rows:%+v", errFuncName, testCases[idx], result, test.Test.Rows, test.Oracle.Rows) + } + if len(result) == 0 && testCases[idx].haveDif { + t.Fatalf("wrong %s work. test case:%+v \nresult should be not empty\n"+ + "mv.Results.Test.Rows:%+v\n"+ + "mv.Results.Oracle.Rows:%+v", errFuncName, testCases[idx], test.Test.Rows, test.Oracle.Rows) + } + if len(result) > 0 { + fmt.Printf("%s from mv.Results of test case:%+v\n", errFuncName, testCases[idx]) + fmt.Println(result.String()) + } + tests-- + } + } +} + +func TestGetCompareInfoSV(t *testing.T) { + var test sv.Results + for idx := range testCases { + tests := 2 + if testCases[idx].diffs > 0 { + tests = 4 + } + for tests > 0 { + test.Test.Rows, test.Oracle.Rows, test.Test.Types, test.Oracle.Types = rndSameRowsSV(testCases[idx].test, testCases[idx].oracle) + // Add names + if len(test.Test.Rows) > 0 { + test.Test.Names = make([]string, len(test.Test.Rows[0])) + for col := range test.Test.Rows[0] { + test.Test.Names[col] = fmt.Sprintf("col%d", col) + } + } + if len(test.Oracle.Rows) > 0 { + test.Oracle.Names = make([]string, len(test.Oracle.Rows[0])) + for col := range test.Oracle.Rows[0] { + test.Oracle.Names[col] = fmt.Sprintf("col%d", col) + } + } + if testCases[idx].diffs > 0 { + if tests%2 == 0 { + corruptRowsSV(&test.Test.Rows, testCases[idx].diffs) + } else { + corruptRowsSV(&test.Oracle.Rows, testCases[idx].diffs) + } + } + result := Info{} + errFuncName := "GetCompareInfoDetailed" + if tests%2 == 0 { + errFuncName = "GetCompareInfoSimple" + result = GetCompareInfoSimple(&test) + } else { + result = GetCompareInfoDetailed(&test) + } + if len(result) > 0 && !testCases[idx].haveDif { + t.Fatalf("wrong %s work. test case:%+v \nresult should be empty, but have:%s\n"+ + "mv.Results.Test.Rows:%+v\n"+ + "mv.Results.Oracle.Rows:%+v", errFuncName, testCases[idx], result, test.Test.Rows, test.Oracle.Rows) + } + if len(result) == 0 && testCases[idx].haveDif { + t.Fatalf("wrong %s work. test case:%+v \nresult should be not empty\n"+ + "mv.Results.Test.Rows:%+v\n"+ + "mv.Results.Oracle.Rows:%+v", errFuncName, testCases[idx], test.Test.Rows, test.Oracle.Rows) + } + if len(result) > 0 { + fmt.Printf("%s from mv.Results of test case:%+v\n", errFuncName, testCases[idx]) + fmt.Println(result.String()) + } + tests-- + } + } +} diff --git a/pkg/store/comp/interface.go b/pkg/store/comp/interface.go new file mode 100644 index 00000000..a1b98e59 --- /dev/null +++ b/pkg/store/comp/interface.go @@ -0,0 +1,44 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package comp + +// Results interface for comparison Test and Oracle rows +type Results interface { + // EqualSingeRow equals fist rows and deletes if equal, returns count of equal rows. + // Equals only the first Oracle and Test rows without `for` cycle. + // Most responses have only one row. + EqualSingeRow() int + + // EasyEqualRowsTest returns count of equal rows into stores simultaneously deletes equal rows. + // Most cases have no difference between Oracle and Test rows, therefore the fastest compare way to compare + // Test and Oracle responses row by row. + // Travels through Test rows. + EasyEqualRowsTest() int + // EasyEqualRowsOracle same as EasyEqualRowsTest, but travels through Oracle rows. + EasyEqualRowsOracle() int + + // EqualRowsTest equals all rows and deletes if equal, returns count of equal rows. + // For cases then EasyEqualRowsTest did not bring full success. + // Travels through Test rows. + EqualRowsTest() int + // EqualRowsOracle same as EqualRowsTest, but travels through Oracle rows. + EqualRowsOracle() int + + LenRowsOracle() int + LenRowsTest() int + + StringAllRows(prefix string) []string + HaveRows() bool +} diff --git a/pkg/store/comp/utils_4test.go b/pkg/store/comp/utils_4test.go new file mode 100644 index 00000000..832d1d5d --- /dev/null +++ b/pkg/store/comp/utils_4test.go @@ -0,0 +1,288 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package comp + +import ( + "time" + + "github.com/gocql/gocql" + "golang.org/x/exp/rand" + + "github.com/scylladb/gemini/pkg/store/mv" + "github.com/scylladb/gemini/pkg/store/sv" + "github.com/scylladb/gemini/pkg/typedef" + "github.com/scylladb/gemini/pkg/utils" +) + +var rnd = rand.New(rand.NewSource(uint64(time.Now().UnixMilli()))) + +type testCase struct { + test, oracle, diffs int + haveDif bool +} + +var testCases = []testCase{ + {test: 0, oracle: 0, diffs: 0, haveDif: false}, + {test: 0, oracle: 1, diffs: 0, haveDif: true}, + {test: 0, oracle: 10, diffs: 0, haveDif: true}, + {test: 1, oracle: 0, diffs: 0, haveDif: true}, + {test: 1, oracle: 1, diffs: 0, haveDif: false}, + {test: 1, oracle: 1, diffs: 1, haveDif: true}, + {test: 1, oracle: 10, diffs: 0, haveDif: true}, + {test: 1, oracle: 10, diffs: 1, haveDif: true}, + {test: 10, oracle: 0, diffs: 0, haveDif: true}, + {test: 10, oracle: 1, diffs: 0, haveDif: true}, + {test: 10, oracle: 1, diffs: 1, haveDif: true}, + {test: 10, oracle: 10, diffs: 0, haveDif: false}, + {test: 10, oracle: 10, diffs: 1, haveDif: true}, + {test: 10, oracle: 10, diffs: 5, haveDif: true}, + {test: 10, oracle: 10, diffs: 10, haveDif: true}, +} + +func corruptRows(rows *mv.RowsMV, diffs int) { + corrupted := make(map[int]struct{}) + corruptCols := [2][]int{ + {0, 1, 2}, + {4, 5, 6}, + } + corruptType := 0 + for diffs > 0 { + row := rnd.Intn(len(*rows)) + _, have := corrupted[row] + if have { + continue + } + if corruptType == 0 { + corruptType = 1 + } else { + corruptType = 0 + } + for _, idx := range corruptCols[corruptType] { + switch (*rows)[row][idx].(type) { + case mv.ColumnRaw: + (*rows)[row][idx], _ = rndSameRaw(19) + case mv.List: + (*rows)[row][idx], _, _, _ = rndSameLists(2, 19) + case mv.Map: + (*rows)[row][idx], _, _, _ = rndSameMaps(2, 19) + case mv.Tuple: + (*rows)[row][idx], _, _, _ = rndSameTuples(2, 19) + case mv.UDT: + (*rows)[row][idx], _, _, _ = rndSameUDTs(2, 19) + } + } + corrupted[row] = struct{}{} + diffs-- + } + _ = corrupted +} + +func corruptRowsSV(rows *sv.RowsSV, diffs int) { + corrupted := make(map[int]struct{}) + corruptCols := [2][]int{ + {0, 1, 2}, + {4, 5, 6}, + } + corruptType := 0 + for diffs > 0 { + row := rnd.Intn(len(*rows)) + _, have := corrupted[row] + if have { + continue + } + if corruptType == 0 { + corruptType = 1 + } else { + corruptType = 0 + } + for _, idx := range corruptCols[corruptType] { + (*rows)[row][idx], _ = rndSameRawSV(19) + } + corrupted[row] = struct{}{} + diffs-- + } + _ = corrupted +} + +func rndSameRowsMV(test, oracle int) (mv.RowsMV, mv.RowsMV, []gocql.TypeInfo, []gocql.TypeInfo) { + testRows := make(mv.RowsMV, test) + oracleRows := make(mv.RowsMV, oracle) + testTypes := make([]gocql.TypeInfo, 20) + oracleTypes := make([]gocql.TypeInfo, 20) + list := oracleRows + if test > oracle { + list = testRows + } + oracle-- + test-- + for range list { + switch { + case oracle < 0: + testRows[test], _, testTypes, _ = rndSameRow(20, 20) + test-- + case test < 0: + _, oracleRows[oracle], _, oracleTypes = rndSameRow(20, 20) + oracle-- + default: + testRows[test], oracleRows[oracle], testTypes, oracleTypes = rndSameRow(20, 20) + test-- + oracle-- + } + } + + return testRows, oracleRows, testTypes, oracleTypes +} + +func rndSameRowsSV(test, oracle int) (sv.RowsSV, sv.RowsSV, []gocql.TypeInfo, []gocql.TypeInfo) { + testRows := make(sv.RowsSV, test) + oracleRows := make(sv.RowsSV, oracle) + testTypes := make([]gocql.TypeInfo, 20) + oracleTypes := make([]gocql.TypeInfo, 20) + list := oracleRows + if test > oracle { + list = testRows + } + for id := range testTypes { + testTypes[id] = allTypes[gocql.TypeText] + oracleTypes[id] = allTypes[gocql.TypeText] + } + oracle-- + test-- + for range list { + switch { + case oracle < 0: + testRows[test], _ = rndSameRowSV(20, 20) + test-- + case test < 0: + _, oracleRows[oracle] = rndSameRowSV(20, 20) + oracle-- + default: + testRows[test], oracleRows[oracle] = rndSameRowSV(20, 20) + test-- + oracle-- + } + } + + return testRows, oracleRows, testTypes, oracleTypes +} + +var allTypes = typedef.GetGoCQLTypeMap() + +func rndSameRow(columns, columnLen int) (mv.RowMV, mv.RowMV, []gocql.TypeInfo, []gocql.TypeInfo) { + out1 := make(mv.RowMV, columns) + out2 := make(mv.RowMV, columns) + out1Type, out2Type := make([]gocql.TypeInfo, columns), make([]gocql.TypeInfo, columns) + for id := 0; id < columns; id++ { + switch id % 5 { + case 0: + out1[id], out2[id] = rndSameRaw(columnLen) + out1Type[id], out2Type[id] = allTypes[gocql.TypeText], allTypes[gocql.TypeText] + case 1: + out1[id], out2[id], out1Type[id], out2Type[id] = rndSameLists(2, columnLen) + case 2: + out1[id], out2[id], out1Type[id], out2Type[id] = rndSameMaps(2, columnLen) + case 3: + out1[id], out2[id], out1Type[id], out2Type[id] = rndSameTuples(2, columnLen) + case 4: + out1[id], out2[id], out1Type[id], out2Type[id] = rndSameUDTs(2, columnLen) + } + } + return out1, out2, out1Type, out2Type +} + +func rndSameRowSV(columns, columnLen int) (sv.RowSV, sv.RowSV) { + out1 := make(sv.RowSV, columns) + out2 := make(sv.RowSV, columns) + for id := range out1 { + out1[id], out2[id] = rndSameRawSV(columnLen) + } + return out1, out2 +} + +func rndSameRawSV(columnLen int) (sv.ColumnRaw, sv.ColumnRaw) { + out1 := []byte(utils.RandString(rnd, columnLen)) + out2 := make(sv.ColumnRaw, columnLen) + copy(out2, out1) + return out1, out2 +} + +func rndSameRaw(columnLen int) (mv.ColumnRaw, mv.ColumnRaw) { + out1 := []byte(utils.RandString(rnd, columnLen)) + out2 := make(mv.ColumnRaw, columnLen) + copy(out2, out1) + return out1, out2 +} + +func rndSameStrings(elems, columnLen int) ([]string, []string) { + out1 := make([]string, elems) + for idx := range out1 { + out1[idx] = utils.RandString(rnd, columnLen) + } + out2 := make([]string, elems) + copy(out2, out1) + return out1, out2 +} + +func rndSameElems(elems, columnLen int) ([]mv.Elem, []mv.Elem) { + out1 := make([]mv.Elem, elems) + out2 := make([]mv.Elem, elems) + for id := range out1 { + tmp1, tmp2 := rndSameRaw(columnLen) + out1[id], out2[id] = &tmp1, &tmp2 + } + return out1, out2 +} + +func rndSameLists(elems, columnLen int) (mv.List, mv.List, gocql.TypeInfo, gocql.TypeInfo) { + var out1, out2 mv.List + outType := gocql.CollectionType{Elem: gocql.NewNativeType(4, gocql.TypeText, "")} + out1, out2 = rndSameElems(elems, columnLen) + return out1, out2, outType, outType +} + +func rndSameMaps(elems, columnLen int) (mv.Map, mv.Map, gocql.TypeInfo, gocql.TypeInfo) { + var out1, out2 mv.Map + outType := gocql.CollectionType{Elem: gocql.NewNativeType(4, gocql.TypeText, ""), Key: gocql.NewNativeType(4, gocql.TypeText, "")} + out1.Keys, out2.Keys = rndSameElems(elems, columnLen) + out1.Values, out2.Values = rndSameElems(elems, columnLen) + return out1, out2, outType, outType +} + +func rndSameTuples(elems, columnLen int) (mv.Tuple, mv.Tuple, gocql.TypeInfo, gocql.TypeInfo) { + var out1, out2 mv.Tuple + outType := gocql.TupleTypeInfo{Elems: make([]gocql.TypeInfo, elems)} + out1, out2 = rndSameElems(elems, columnLen) + for idx := range out1 { + outType.Elems[idx] = gocql.NewNativeType(4, gocql.TypeText, "") + } + return out1, out2, outType, outType +} + +func rndSameUDTs(elems, columnLen int) (mv.UDT, mv.UDT, gocql.TypeInfo, gocql.TypeInfo) { + var out1, out2 mv.UDT + outType := gocql.UDTTypeInfo{Elements: make([]gocql.UDTField, elems)} + outType.NativeType = gocql.NewNativeType(4, gocql.TypeUDT, "") + out1.Names, out2.Names = rndSameStrings(elems, columnLen) + out1.Values, out2.Values = rndSameElems(elems, columnLen) + for idx := range out1.Names { + outType.Elements[idx] = gocql.UDTField{ + Name: out1.Names[idx], + Type: gocql.NewNativeType(4, gocql.TypeText, ""), + } + } + return out1, out2, outType, outType +} diff --git a/pkg/store/cqlstore.go b/pkg/store/cqlstore.go index 82a99868..34f80423 100644 --- a/pkg/store/cqlstore.go +++ b/pkg/store/cqlstore.go @@ -27,6 +27,8 @@ import ( "github.com/scylladb/gocqlx/v2/qb" "go.uber.org/zap" + mv "github.com/scylladb/gemini/pkg/store/mv" + sv "github.com/scylladb/gemini/pkg/store/sv" "github.com/scylladb/gemini/pkg/typedef" ) @@ -90,11 +92,25 @@ func (cs *cqlStore) doMutate(ctx context.Context, builder qb.Builder, ts time.Ti return nil } -func (cs *cqlStore) load(ctx context.Context, builder qb.Builder, values []interface{}) (result []map[string]interface{}, err error) { +func (cs *cqlStore) loadSV(ctx context.Context, builder qb.Builder, values []interface{}) (sv.Result, error) { query, _ := builder.ToCql() iter := cs.session.Query(query, values...).WithContext(ctx).Iter() cs.ops.WithLabelValues(cs.system, opType(builder)).Inc() - return loadSet(iter), iter.Close() + return sv.GetResult(iter), iter.Close() +} + +func (cs *cqlStore) loadMV(ctx context.Context, builder qb.Builder, values []interface{}) (mv.Result, error) { + query, _ := builder.ToCql() + iter := cs.session.Query(query, values...).WithContext(ctx).Iter() + cs.ops.WithLabelValues(cs.system, opType(builder)).Inc() + return mv.GetResult(iter), iter.Close() +} + +func (cs *cqlStore) loadVerCheck(ctx context.Context, builder qb.Builder, values []interface{}) (mv.Result, error) { + query, _ := builder.ToCql() + iter := cs.session.Query(query, values...).WithContext(ctx).Iter() + cs.ops.WithLabelValues(cs.system, opType(builder)).Inc() + return mv.GetResultWithVerCheck(iter), iter.Close() } func (cs cqlStore) close() error { diff --git a/pkg/store/helpers.go b/pkg/store/helpers.go deleted file mode 100644 index c95546d5..00000000 --- a/pkg/store/helpers.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2019 ScyllaDB -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package store - -import ( - "fmt" - "math/big" - "strings" - "time" - - "github.com/gocql/gocql" - - "github.com/scylladb/gemini/pkg/typedef" -) - -func pks(t *typedef.Table, rows []map[string]interface{}) []string { - var keySet []string - for _, row := range rows { - keys := make([]string, 0, len(t.PartitionKeys)+len(t.ClusteringKeys)) - keys = extractRowValues(keys, t.PartitionKeys, row) - keys = extractRowValues(keys, t.ClusteringKeys, row) - keySet = append(keySet, strings.Join(keys, ", ")) - } - return keySet -} - -func extractRowValues(values []string, columns typedef.Columns, row map[string]interface{}) []string { - for _, pk := range columns { - values = append(values, fmt.Sprintf(pk.Name+"=%v", row[pk.Name])) - } - return values -} - -func lt(mi, mj map[string]interface{}) bool { - switch mis := mi["pk0"].(type) { - case []byte: - mjs, _ := mj["pk0"].([]byte) - return string(mis) < string(mjs) - case string: - mjs, _ := mj["pk0"].(string) - return mis < mjs - case int: - mjs, _ := mj["pk0"].(int) - return mis < mjs - case int8: - mjs, _ := mj["pk0"].(int8) - return mis < mjs - case int16: - mjs, _ := mj["pk0"].(int16) - return mis < mjs - case int32: - mjs, _ := mj["pk0"].(int32) - return mis < mjs - case int64: - mjs, _ := mj["pk0"].(int64) - return mis < mjs - case gocql.UUID: - mjs, _ := mj["pk0"].(gocql.UUID) - return mis.String() < mjs.String() - case time.Time: - mjs, _ := mj["pk0"].(time.Time) - return mis.UnixNano() < mjs.UnixNano() - case *big.Int: - mjs, _ := mj["pk0"].(*big.Int) - return mis.Cmp(mjs) < 0 - case nil: - return true - default: - msg := fmt.Sprintf("unhandled type %T\n", mis) - time.Sleep(time.Second) - panic(msg) - } -} - -func loadSet(iter *gocql.Iter) []map[string]interface{} { - var rows []map[string]interface{} - for { - row := make(map[string]interface{}) - if !iter.MapScan(row) { - break - } - rows = append(rows, row) - } - return rows -} diff --git a/pkg/store/mv/c_list.go b/pkg/store/mv/c_list.go new file mode 100644 index 00000000..fcf8eefc --- /dev/null +++ b/pkg/store/mv/c_list.go @@ -0,0 +1,198 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +import ( + "fmt" + "reflect" + + "github.com/gocql/gocql" +) + +type List []Elem + +func (l List) ToString(colInfo gocql.TypeInfo) string { + out := "set<" + if colInfo.Type() == gocql.TypeList { + out = "list<" + } + if len(l) == 0 { + return out + ">" + } + listInfo := colInfo.(gocql.CollectionType) + for idx := range l { + out += fmt.Sprintf("%d:%s;", idx, l[idx].ToString(listInfo.Elem)) + } + out = out[:len(out)-1] + return out + ">" +} + +func (l List) ToStringRaw() string { + out := "set||list<" + if len(l) == 0 { + return out + ">" + } + for idx := range l { + out += fmt.Sprintf("%d:%s;", idx, l[idx].ToStringRaw()) + } + out = out[:len(out)-1] + return out + ">" +} + +func (l List) EqualColumn(colT interface{}) bool { + l2, ok := colT.(List) + if len(l) != len(l2) || !ok { + return false + } + if len(l) == 0 { + return true + } + for idx := range l { + if !l[idx].EqualElem(l2[idx]) { + return false + } + } + return true +} + +func (l List) EqualElem(colT interface{}) bool { + l2, ok := colT.(*List) + if len(l) != len(*l2) || !ok { + return false + } + if len(l) == 0 { + return true + } + for idx := range l { + if !l[idx].EqualElem((*l2)[idx]) { + return false + } + } + return true +} + +func (l *List) UnmarshalCQL(colInfo gocql.TypeInfo, data []byte) error { + if len(data) < 1 { + return nil + } + col, ok := colInfo.(gocql.CollectionType) + if !ok { + return fmt.Errorf("%+v is unsupported type to unmarshal in list/set", reflect.TypeOf(colInfo)) + } + if colInfo.Version() <= protoVersion2 { + if len(data) < 2 { + return ErrorUnmarshalEOF + } + return l.unmarshalOld(col, data) + } + if len(data) < 4 { + return ErrorUnmarshalEOF + } + return l.unmarshalNew(col, data) +} + +func (l *List) unmarshalNew(colInfo gocql.CollectionType, data []byte) error { + var err error + listLen := readLen(data) + data = data[4:] + switch listLen { + case 0: + case 1: + var elemLen int32 + if len(data) < 4 { + return ErrorUnmarshalEOF + } + elemLen = readLen(data) + 4 + if len(data) < int(elemLen) { + return ErrorUnmarshalEOF + } + if err = (*l)[0].UnmarshalCQL(colInfo.Elem, data[4:elemLen]); err != nil { + return err + } + default: + var elemLen int32 + *l = append(*l, make(List, listLen-1)...) + for idx := range *l { + if idx > 0 { + (*l)[idx] = (*l)[0].NewSameElem() + } + if len(data) < 4 { + return ErrorUnmarshalEOF + } + elemLen = readLen(data) + 4 + if len(data) < int(elemLen) { + return ErrorUnmarshalEOF + } + if err = (*l)[idx].UnmarshalCQL(colInfo.Elem, data[4:elemLen]); err != nil { + return err + } + data = data[elemLen:] + } + } + return nil +} + +func (l *List) unmarshalOld(colInfo gocql.CollectionType, data []byte) error { + var err error + listLen := readLenOld(data) + data = data[2:] + switch listLen { + case 0: + case 1: + var elemLen int16 + if len(data) < 2 { + return ErrorUnmarshalEOF + } + elemLen = readLenOld(data) + 2 + if len(data) < int(elemLen) { + return ErrorUnmarshalEOF + } + if err = (*l)[0].UnmarshalCQL(colInfo.Elem, data[2:elemLen]); err != nil { + return err + } + default: + var elemLen int16 + *l = append(*l, make(List, listLen-1)...) + for idx := range *l { + if idx > 0 { + (*l)[idx] = (*l)[0].NewSameElem() + } + if len(data) < 2 { + return ErrorUnmarshalEOF + } + elemLen = readLenOld(data) + 2 + if len(data) < int(elemLen) { + return ErrorUnmarshalEOF + } + if err = (*l)[idx].UnmarshalCQL(colInfo.Elem, data[2:elemLen]); err != nil { + return err + } + data = data[elemLen:] + } + } + return nil +} + +func (l List) NewSameColumn() Column { + return List{l[0].NewSameElem()} +} + +func (l List) ToUnmarshal() interface{} { + return &l +} + +func (l List) NewSameElem() Elem { + return &List{l[0].NewSameElem()} +} diff --git a/pkg/store/mv/c_list_test.go b/pkg/store/mv/c_list_test.go new file mode 100644 index 00000000..0cb90c70 --- /dev/null +++ b/pkg/store/mv/c_list_test.go @@ -0,0 +1,100 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package mv + +import ( + "reflect" + "testing" + + "github.com/gocql/gocql" +) + +func TestList_UnmarshalCQL(t *testing.T) { + errorMsg := "wrong List.UnmarshalCQL work:" + var err error + testsCount := 1000 + old := true + for i := 0; i < testsCount; i++ { + elems := 20 + maxElemLen := 10 + if i == 0 { + elems = 0 + maxElemLen = 0 + } + if i/2 > testsCount/2 { + old = false + elems = 0 + maxElemLen = 0 + } + expected, data := rndDataElems(elems, maxElemLen, old, true) + // List initialization. + testColumn := make(List, 1) + testColumn[0] = &ColumnRaw{} + // Unmarshall. + if old { + err = testColumn.unmarshalOld(gocql.CollectionType{}, data) + } else { + err = testColumn.unmarshalNew(gocql.CollectionType{}, data) + } + + // Check results. + if err != nil { + t.Fatalf("%s error:%s", errorMsg, err) + } + // With correction needed, because List and Map initialization required fist elem + if len(expected) == 0 && len(testColumn) == 1 && testColumn[0].EqualColumn(ColumnRaw{}) { + continue + } + if len(testColumn) != len(expected) { + t.Fatalf("%s\nreceived len:%d \nexpected len:%d", errorMsg, len(testColumn), len(expected)) + } + for idx := range testColumn { + if !reflect.DeepEqual(expected[idx], testColumn[idx]) { + t.Fatalf("%s\nreceived:%+v \nexpected:%+v", errorMsg, testColumn[idx], expected[idx]) + } + } + } +} + +func TestList_Equal(t *testing.T) { + testColumn1 := make(List, 0) + testColumn2 := make(List, 0) + for i := range testCeases { + testColumn1, testColumn2 = rndSameElems(testCeases[i].elems, testCeases[i].elemLen) + // EqualColumn test on equal + if !testColumn1.EqualColumn(testColumn2) { + t.Fatal("List.EqualColumn should return true") + } + // EqualElem test on equal + if !testColumn1.EqualElem(&testColumn2) { + t.Fatal("List.EqualElem should return true") + } + testColumn2 = []Elem{ + &ColumnRaw{1, 2, 3}, + } + // EqualColumn test on unequal + if testColumn1.EqualColumn(testColumn2) { + t.Fatal("List.EqualColumn should return false") + } + // EqualElem test on unequal + if testColumn1.EqualElem(&testColumn2) { + t.Fatal("List.EqualElem should return false") + } + } + _ = testColumn1 + _ = testColumn2 +} diff --git a/pkg/store/mv/c_map.go b/pkg/store/mv/c_map.go new file mode 100644 index 00000000..34a40b09 --- /dev/null +++ b/pkg/store/mv/c_map.go @@ -0,0 +1,247 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +import ( + "fmt" + "reflect" + + "github.com/gocql/gocql" +) + +type Map struct { + Keys []Elem + Values []Elem +} + +func (m Map) ToString(colInfo gocql.TypeInfo) string { + out := "map(" + if len(m.Keys) == 0 { + return out + ">" + } + mapInfo := colInfo.(gocql.CollectionType) + for idx := range m.Keys { + out += fmt.Sprintf("%s:%s;", m.Keys[idx].ToString(mapInfo.Key), m.Values[idx].ToString(mapInfo.Elem)) + } + out = out[:len(out)-1] + return out + ")" +} + +func (m Map) ToStringRaw() string { + out := "map<" + if len(m.Keys) == 0 { + return out + ">" + } + for idx := range m.Keys { + out += fmt.Sprintf("key%d%s:value%s;", idx, m.Keys[idx].ToStringRaw(), m.Values[idx].ToStringRaw()) + } + out = out[:len(out)-1] + return out + ")" +} + +func (m Map) EqualColumn(colT interface{}) bool { + m2, ok := colT.(Map) + if len(m.Keys) != len(m2.Keys) || len(m.Values) != len(m2.Values) || !ok { + return false + } + if len(m.Keys) == 0 { + return true + } + for idx := range m.Keys { + if !m.Keys[idx].EqualElem(m2.Keys[idx]) { + return false + } + if !m.Values[idx].EqualElem(m2.Values[idx]) { + return false + } + } + return true +} + +func (m Map) EqualElem(colT interface{}) bool { + m2, ok := colT.(*Map) + if len(m.Keys) != len(m2.Keys) || len(m.Values) != len(m2.Values) || !ok { + return false + } + if len(m.Keys) == 0 { + return true + } + for idx := range m.Keys { + if !m.Keys[idx].EqualElem(m2.Keys[idx]) { + return false + } + if !m.Values[idx].EqualElem(m2.Values[idx]) { + return false + } + } + return true +} + +func (m *Map) UnmarshalCQL(colInfo gocql.TypeInfo, data []byte) error { + col, ok := colInfo.(gocql.CollectionType) + if !ok { + return fmt.Errorf("%+v is unsupported type to unmarshal in map", reflect.TypeOf(colInfo)) + } + if colInfo.Version() <= protoVersion2 { + if len(data) < 2 { + return ErrorUnmarshalEOF + } + return m.oldUnmarshalCQL(col, data) + } + if len(data) < 4 { + return ErrorUnmarshalEOF + } + return m.newUnmarshalCQL(col, data) +} + +func (m *Map) oldUnmarshalCQL(mapInfo gocql.CollectionType, data []byte) error { + var err error + mapLen := readLenOld(data) + data = data[2:] + switch mapLen { + case 0: + case 1: + var keyLen, valLen int16 + if len(data) < 4 { + return ErrorUnmarshalEOF + } + keyLen = readLenOld(data) + 2 + if len(data) < int(keyLen) { + return ErrorUnmarshalEOF + } + if err = m.Keys[0].UnmarshalCQL(mapInfo.Key, data[2:keyLen]); err != nil { + return err + } + data = data[keyLen:] + + valLen = readLenOld(data) + 2 + if len(data) < int(valLen) { + return ErrorUnmarshalEOF + } + if err = m.Values[0].UnmarshalCQL(mapInfo.Elem, data[2:valLen]); err != nil { + return err + } + default: + var keyLen, valLen int16 + m.Keys = append(m.Keys, make([]Elem, mapLen-1)...) + m.Values = append(m.Values, make([]Elem, mapLen-1)...) + for idx := range m.Keys { + if idx > 0 { + m.Keys[idx] = m.Keys[0].NewSameElem() + m.Values[idx] = m.Values[0].NewSameElem() + } + if len(data) < 4 { + return ErrorUnmarshalEOF + } + keyLen = readLenOld(data) + 2 + if len(data) < int(keyLen) { + return ErrorUnmarshalEOF + } + if err = m.Keys[idx].UnmarshalCQL(mapInfo.Key, data[2:keyLen]); err != nil { + return err + } + data = data[keyLen:] + + valLen = readLenOld(data) + 2 + if len(data) < int(valLen) { + return ErrorUnmarshalEOF + } + if err = m.Values[idx].UnmarshalCQL(mapInfo.Elem, data[2:valLen]); err != nil { + return err + } + data = data[valLen:] + } + } + return nil +} + +func (m *Map) newUnmarshalCQL(mapInfo gocql.CollectionType, data []byte) error { + var err error + mapLen := readLen(data) + data = data[4:] + switch mapLen { + case 0: + case 1: + var keyLen, valLen int32 + if len(data) < 8 { + return ErrorUnmarshalEOF + } + keyLen = readLen(data) + 4 + if len(data) < int(keyLen) { + return ErrorUnmarshalEOF + } + if err = m.Keys[0].UnmarshalCQL(mapInfo.Key, data[4:keyLen]); err != nil { + return err + } + data = data[keyLen:] + + valLen = readLen(data) + 4 + if len(data) < int(valLen) { + return ErrorUnmarshalEOF + } + if err = m.Values[0].UnmarshalCQL(mapInfo.Elem, data[4:valLen]); err != nil { + return err + } + default: + var keyLen, valLen int32 + m.Keys = append(m.Keys, make([]Elem, mapLen-1)...) + m.Values = append(m.Values, make([]Elem, mapLen-1)...) + for idx := range m.Keys { + if idx > 0 { + m.Keys[idx] = m.Keys[0].NewSameElem() + m.Values[idx] = m.Values[0].NewSameElem() + } + if len(data) < 8 { + return ErrorUnmarshalEOF + } + keyLen = readLen(data) + 4 + if len(data) < int(keyLen) { + return ErrorUnmarshalEOF + } + if err = m.Keys[idx].UnmarshalCQL(mapInfo.Key, data[4:keyLen]); err != nil { + return err + } + data = data[keyLen:] + + valLen = readLen(data) + 4 + if len(data) < int(valLen) { + return ErrorUnmarshalEOF + } + if err = m.Values[idx].UnmarshalCQL(mapInfo.Elem, data[4:valLen]); err != nil { + return err + } + data = data[valLen:] + } + } + return nil +} + +func (m Map) NewSameColumn() Column { + return Map{ + Keys: []Elem{m.Keys[0].NewSameElem()}, + Values: []Elem{m.Values[0].NewSameElem()}, + } +} + +func (m Map) ToUnmarshal() interface{} { + return &m +} + +func (m *Map) NewSameElem() Elem { + return &Map{ + Keys: []Elem{m.Keys[0].NewSameElem()}, + Values: []Elem{m.Values[0].NewSameElem()}, + } +} diff --git a/pkg/store/mv/c_map_test.go b/pkg/store/mv/c_map_test.go new file mode 100644 index 00000000..36ab1b94 --- /dev/null +++ b/pkg/store/mv/c_map_test.go @@ -0,0 +1,124 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package mv + +import ( + "reflect" + "testing" + + "github.com/gocql/gocql" +) + +func TestMap_UnmarshalCQL(t *testing.T) { + errorMsg := "wrong Map.UnmarshalCQL work:" + testColumn := Map{} + var err error + testsCount := 1000 + old := true + for i := 0; i < testsCount; i++ { + elems := 20 + maxElemLen := 10 + if i == 0 { + elems = 0 + maxElemLen = 0 + } + if i/2 > testsCount/2 { + old = false + elems = 0 + maxElemLen = 0 + } + expectedKeys, expectedValues, data := rndDataElemsMap(elems, maxElemLen, old) + // Map initialization. + testColumn.Keys = make([]Elem, 1) + testColumn.Values = make([]Elem, 1) + testColumn.Keys[0] = &ColumnRaw{} + testColumn.Values[0] = &ColumnRaw{} + // Unmarshall. + if old { + err = testColumn.oldUnmarshalCQL(gocql.CollectionType{}, data) + } else { + err = testColumn.newUnmarshalCQL(gocql.CollectionType{}, data) + } + + // Check results. + if err != nil { + t.Fatalf("%s error:%s", errorMsg, err) + } + // Check Keys + if len(testColumn.Keys) != len(expectedKeys) { + t.Fatalf("%s\nreceived keys len:%d \nexpected keys len:%d", errorMsg, len(testColumn.Keys), len(expectedKeys)) + } + for idx := range testColumn.Keys { + if !reflect.DeepEqual(expectedKeys[idx], testColumn.Keys[idx]) { + t.Fatalf("%s\nreceived keys:%+v \nexpected keys:%+v", errorMsg, testColumn.Keys[idx], expectedKeys[idx]) + } + } + // Check Values + if len(testColumn.Values) != len(expectedValues) { + t.Fatalf("%s\nreceived keys len:%d \nexpected keys len:%d", errorMsg, len(testColumn.Values), len(expectedValues)) + } + for idx := range testColumn.Values { + if !reflect.DeepEqual(expectedValues[idx], testColumn.Values[idx]) { + t.Fatalf("%s\nreceived keys:%+v \nexpected keys:%+v", errorMsg, testColumn.Values[idx], expectedValues[idx]) + } + } + } +} + +func TestMap_Equal(t *testing.T) { + var testColumn1, testColumn2 Map + + for i := range testCeases { + testColumn1.Keys, testColumn2.Keys = rndSameElems(testCeases[i].elems, testCeases[i].elemLen) + testColumn1.Values, testColumn2.Values = rndSameElems(testCeases[i].elems, testCeases[i].elemLen) + // EqualColumn test on equal + if !testColumn1.EqualColumn(testColumn2) { + t.Fatal("Map.EqualColumn should return true") + } + // EqualElem test on equal + if !testColumn1.EqualElem(&testColumn2) { + t.Fatal("Map.EqualElem should return true") + } + + // Corrupt values + testColumn2.Values = []Elem{ + &ColumnRaw{1, 2, 3}, + } + // EqualColumn test on unequal + if testColumn1.EqualColumn(testColumn2) { + t.Fatal("Map.EqualColumn should return false") + } + // EqualElem test on unequal + if testColumn1.EqualElem(&testColumn2) { + t.Fatal("Map.EqualElem should return false") + } + + // Corrupt keys + testColumn1.Values, testColumn2.Values = rndSameElems(testCeases[i].elems, testCeases[i].elemLen) + testColumn2.Keys = []Elem{ + &ColumnRaw{1, 2, 3}, + } + // EqualColumn test on unequal + if testColumn1.EqualColumn(testColumn2) { + t.Fatal("Map.EqualColumn should return false") + } + // EqualElem test on unequal + if testColumn1.EqualElem(&testColumn2) { + t.Fatal("Map.EqualElem should return false") + } + } +} diff --git a/pkg/store/mv/c_raw.go b/pkg/store/mv/c_raw.go new file mode 100644 index 00000000..1e7a5580 --- /dev/null +++ b/pkg/store/mv/c_raw.go @@ -0,0 +1,111 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +import ( + "fmt" + + "github.com/gocql/gocql" +) + +// ColumnRaw for most cases. +type ColumnRaw []byte + +func (col ColumnRaw) ToString(colInfo gocql.TypeInfo) string { + if len(col) == 0 { + return "" + } + tmpVal := colInfo.New() + if err := gocql.Unmarshal(colInfo, col, tmpVal); err != nil { + panic(err) + } + out := fmt.Sprintf("%v", dereference(tmpVal)) + // Add name of the type for complex and collections types + switch colInfo.Type() { + case gocql.TypeList: + out = fmt.Sprintf("list<%s>", out) + case gocql.TypeSet: + out = fmt.Sprintf("set<%s>", out) + case gocql.TypeMap: + out = fmt.Sprintf("map<%s>", out) + case gocql.TypeTuple: + out = fmt.Sprintf("tuple<%s>", out) + case gocql.TypeUDT: + out = fmt.Sprintf("udt<%s>", out) + } + return out +} + +func (col ColumnRaw) ToStringRaw() string { + return fmt.Sprint(col) +} + +func (col ColumnRaw) EqualColumn(colT interface{}) bool { + col2, ok := colT.(ColumnRaw) + if len(col) != len(col2) || !ok { + // Columns len are different - means columns are unequal + return false + } + if len(col) < 1 { + // Columns len==0 and same - means columns are equal + // Conditions "<" or ">" works faster that "==" + return true + } + for idx := range col { + if col[idx] != col2[idx] { + return false + } + } + return true +} + +func (col ColumnRaw) EqualElem(colT interface{}) bool { + col2, ok := colT.(*ColumnRaw) + if len(col) != len(*col2) || !ok { + // Columns len are different - means columns are unequal + return false + } + if len(col) < 1 { + // Columns len==0 and same - means columns are equal + // Conditions "<" or ">" works faster that "==" + return true + } + for idx := range col { + if col[idx] != (*col2)[idx] { + return false + } + } + return true +} + +func (col ColumnRaw) NewSameColumn() Column { + return ColumnRaw{} +} + +func (col ColumnRaw) ToUnmarshal() interface{} { + return &col +} + +func (col ColumnRaw) NewSameElem() Elem { + return &ColumnRaw{} +} + +func (col *ColumnRaw) UnmarshalCQL(_ gocql.TypeInfo, data []byte) error { + if len(data) > 0 { + // Puts data without copying + *col = data + } + return nil +} diff --git a/pkg/store/mv/c_raw_test.go b/pkg/store/mv/c_raw_test.go new file mode 100644 index 00000000..e48529b5 --- /dev/null +++ b/pkg/store/mv/c_raw_test.go @@ -0,0 +1,74 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package mv + +import ( + "reflect" + "testing" + + "github.com/scylladb/gemini/pkg/utils" +) + +func TestColumnRaw_UnmarshalCQL(t *testing.T) { + errorMsg := "wrong ColumnRaw.UnmarshalCQL work:" + testColumn := make(ColumnRaw, 0) + + testsCount := 1000 + for i := 0; i < testsCount; i++ { + expected := utils.RandBytes(rnd, rnd.Intn(1000)) + if i == 0 { + expected = ColumnRaw{} + } + _ = testColumn.UnmarshalCQL(nil, expected) + if !reflect.DeepEqual(expected, ([]byte)(testColumn)) { + t.Fatalf("%s\nreceived:%+v \nexpected:%+v", errorMsg, testColumn, expected) + } + testColumn = make(ColumnRaw, 0) + } +} + +func TestColumnRaw_Equal(t *testing.T) { + testColumn1 := make(ColumnRaw, 0) + testColumn2 := make(ColumnRaw, 0) + tests := []ColumnRaw{ + utils.RandBytes(rnd, rnd.Intn(1000)), + []byte{}, + } + for i := range tests { + testColumn1 = tests[i] + testColumn2 = tests[i] + // EqualColumn test on equal + if !testColumn1.EqualColumn(testColumn2) { + t.Fatal("ColumnRaw.EqualColumn should return true") + } + // EqualElem test on equal + if !testColumn1.EqualElem(&testColumn2) { + t.Fatal("ColumnRaw.EqualElem should return true") + } + testColumn2 = utils.RandBytes(rnd, rnd.Intn(30)) + // EqualColumn test on unequal + if testColumn1.EqualColumn(testColumn2) { + t.Fatal("ColumnRaw.EqualColumn should return false") + } + // EqualElem test on unequal + if testColumn1.EqualElem(&testColumn2) { + t.Fatal("ColumnRaw.EqualElem should return false") + } + } + _ = testColumn1 + _ = testColumn2 +} diff --git a/pkg/store/mv/c_tuple.go b/pkg/store/mv/c_tuple.go new file mode 100644 index 00000000..27e7164c --- /dev/null +++ b/pkg/store/mv/c_tuple.go @@ -0,0 +1,126 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +import ( + "fmt" + "reflect" + + "github.com/gocql/gocql" +) + +type Tuple []Elem + +func (t Tuple) ToString(colInfo gocql.TypeInfo) string { + out := "tuple<" + if len(t) == 0 { + return out + ">" + } + tuple := colInfo.(gocql.TupleTypeInfo) + for i, elem := range tuple.Elems { + out += fmt.Sprintf("%d:%s;", i, t[i].ToString(elem)) + } + out = out[:len(out)-1] + return out + ">" +} + +func (t Tuple) ToStringRaw() string { + out := "tuple<" + if len(t) == 0 { + return out + ">" + } + for i := range t { + out += fmt.Sprintf("%d:%s;", i, t[i].ToStringRaw()) + } + out = out[:len(out)-1] + return out + ">" +} + +func (t Tuple) EqualColumn(colT interface{}) bool { + t2, ok := colT.(Tuple) + if len(t) != len(t2) || !ok { + return false + } + if len(t) == 0 { + return true + } + for idx := range t { + if !t[idx].EqualElem(t2[idx]) { + return false + } + } + return true +} + +func (t Tuple) EqualElem(colT interface{}) bool { + t2, ok := colT.(*Tuple) + if len(t) != len(*t2) || !ok { + return false + } + if len(t) == 0 { + return true + } + for idx := range t { + if !t[idx].EqualElem((*t2)[idx]) { + return false + } + } + return true +} + +func (t *Tuple) UnmarshalCQL(colInfo gocql.TypeInfo, data []byte) error { + if len(data) == 0 { + return nil + } + tuple, ok := colInfo.(gocql.TupleTypeInfo) + if !ok { + return fmt.Errorf("%+v is unsupported type to unmarshal in tuple", reflect.TypeOf(colInfo)) + } + for idx := range *t { + if len(data) < 4 { + return ErrorUnmarshalEOF + } + elemLen := readLen(data) + 4 + if int32(len(data)) < elemLen { + return ErrorUnmarshalEOF + } + err := (*t)[idx].UnmarshalCQL(tuple.Elems[idx], data[4:elemLen]) + if err != nil { + return err + } + data = data[elemLen:] + } + return nil +} + +func (t Tuple) NewSameColumn() Column { + out := make(Tuple, len(t)) + for idx := range t { + out[idx] = t[idx].NewSameElem() + } + return out +} + +func (t Tuple) NewSameElem() Elem { + out := make(Tuple, len(t)) + for idx := range t { + out[idx] = t[idx].NewSameElem() + } + return &out +} + +func (t Tuple) ToUnmarshal() interface{} { + return &t +} diff --git a/pkg/store/mv/c_tuple_test.go b/pkg/store/mv/c_tuple_test.go new file mode 100644 index 00000000..69b64058 --- /dev/null +++ b/pkg/store/mv/c_tuple_test.go @@ -0,0 +1,88 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package mv + +import ( + "reflect" + "testing" + + "github.com/gocql/gocql" +) + +func TestTuple_UnmarshalCQL(t *testing.T) { + errorMsg := "wrong Tuple.UnmarshalCQL work:" + + testsCount := 1000 + for i := 0; i < testsCount; i++ { + elems := 20 + maxElemLen := 10 + if i == 0 { + elems = 0 + maxElemLen = 0 + } + expected, data := rndDataElems(elems, maxElemLen, false, false) + // Tuple initialization. + testColumn := make(Tuple, elems) + for idx := range testColumn { + testColumn[idx] = &ColumnRaw{} + } + // Unmarshall. + err := testColumn.UnmarshalCQL(gocql.TupleTypeInfo{Elems: make([]gocql.TypeInfo, elems)}, data) + // Check results. + if err != nil { + t.Fatalf("%s error:%s", errorMsg, err) + } + if len(testColumn) != len(expected) { + t.Fatalf("%s\nreceived len:%d \nexpected len:%d", errorMsg, len(testColumn), len(expected)) + } + for idx := range testColumn { + if !reflect.DeepEqual(expected[idx], testColumn[idx]) { + t.Fatalf("%s\nreceived:%+v \nexpected:%+v", errorMsg, testColumn[idx], expected[idx]) + } + } + testColumn = make(Tuple, 0) + } +} + +func TestTuple_Equal(t *testing.T) { + testColumn1 := make(Tuple, 0) + testColumn2 := make(Tuple, 0) + for i := range testCeases { + testColumn1, testColumn2 = rndSameElems(testCeases[i].elems, testCeases[i].elemLen) + // EqualColumn test on equal + if !testColumn1.EqualColumn(testColumn2) { + t.Fatal("Tuple.EqualColumn should return true") + } + // EqualElem test on equal + if !testColumn1.EqualElem(&testColumn2) { + t.Fatal("Tuple.EqualElem should return true") + } + testColumn2 = []Elem{ + &ColumnRaw{1, 2, 3}, + } + // EqualColumn test on unequal + if testColumn1.EqualColumn(testColumn2) { + t.Fatal("Tuple.EqualColumn should return false") + } + // EqualElem test on unequal + if testColumn1.EqualElem(&testColumn2) { + t.Fatal("Tuple.EqualElem should return false") + } + } + _ = testColumn1 + _ = testColumn2 +} diff --git a/pkg/store/mv/c_udt.go b/pkg/store/mv/c_udt.go new file mode 100644 index 00000000..d3964c68 --- /dev/null +++ b/pkg/store/mv/c_udt.go @@ -0,0 +1,152 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +import ( + "fmt" + "reflect" + + "github.com/gocql/gocql" +) + +type UDT struct { + Names []string + Values []Elem +} + +func (u UDT) ToString(colInfo gocql.TypeInfo) string { + out := "udt<" + if len(u.Names) == 0 { + return out + ">" + } + udt := colInfo.(gocql.UDTTypeInfo) + + for idx := range u.Values { + out += u.pairToString(idx, udt.Elements[idx].Type) + } + out = out[:len(out)-1] + return out + ">" +} + +func (u UDT) pairToString(idx int, colType gocql.TypeInfo) string { + return fmt.Sprintf("%s:%s;", u.Names[idx], u.Values[idx].ToString(colType)) +} + +func (u UDT) ToStringRaw() string { + out := "udt<" + if len(u.Names) == 0 { + return out + ">" + } + for idx := range u.Values { + out += u.pairToStringRaw(idx) + } + out = out[:len(out)-1] + return out + ">" +} + +func (u UDT) pairToStringRaw(idx int) string { + return fmt.Sprintf("%s:%s;", u.Names[idx], u.Values[idx].ToStringRaw()) +} + +func (u UDT) EqualColumn(colT interface{}) bool { + u2, ok := colT.(UDT) + if len(u.Values) != len(u2.Values) || len(u.Names) != len(u2.Names) || !ok { + return false + } + if len(u.Values) == 0 && len(u.Names) == 0 { + return true + } + for idx := range u.Values { + if !u.Values[idx].EqualElem(u2.Values[idx]) { + return false + } + } + for idx := range u.Names { + if u.Names[idx] != u2.Names[idx] { + return false + } + } + return true +} + +func (u UDT) EqualElem(colT interface{}) bool { + u2, ok := colT.(*UDT) + if len(u.Values) != len(u2.Values) || len(u.Names) != len(u2.Names) || !ok { + return false + } + if len(u.Values) == 0 && len(u.Names) == 0 { + return true + } + for idx := range u.Values { + if !u.Values[idx].EqualElem(u2.Values[idx]) { + return false + } + } + for idx := range u.Names { + if u.Names[idx] != u2.Names[idx] { + return false + } + } + return true +} + +func (u *UDT) UnmarshalCQL(colInfo gocql.TypeInfo, data []byte) error { + if len(data) == 0 { + return nil + } + udt, ok := colInfo.(gocql.UDTTypeInfo) + if !ok { + return fmt.Errorf("%+v is unsupported type to unmarshal in udt", reflect.TypeOf(colInfo)) + } + for idx := range u.Values { + switch len(data) { + case 0: + return nil + case 1, 2, 3: + return ErrorUnmarshalEOF + default: + elemLen := readLen(data) + 4 + if int32(len(data)) < elemLen { + return ErrorUnmarshalEOF + } + err := u.Values[idx].UnmarshalCQL(udt.Elements[idx].Type, data[4:elemLen]) + if err != nil { + return err + } + data = data[elemLen:] + } + } + return nil +} + +func (u UDT) NewSameColumn() Column { + out := UDT{Names: u.Names, Values: make([]Elem, len(u.Values))} + for idx := range u.Values { + out.Values[idx] = u.Values[idx].NewSameElem() + } + return out +} + +func (u UDT) NewSameElem() Elem { + out := UDT{Names: u.Names, Values: make([]Elem, len(u.Values))} + for idx := range u.Values { + out.Values[idx] = u.Values[idx].NewSameElem() + } + return &out +} + +func (u UDT) ToUnmarshal() interface{} { + return &u +} diff --git a/pkg/store/mv/c_udt_test.go b/pkg/store/mv/c_udt_test.go new file mode 100644 index 00000000..f9dafafc --- /dev/null +++ b/pkg/store/mv/c_udt_test.go @@ -0,0 +1,103 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package mv + +import ( + "reflect" + "testing" + + "github.com/gocql/gocql" +) + +func TestUDT_UnmarshalCQL(t *testing.T) { + errorMsg := "wrong UDT.UnmarshalCQL work:" + testColumn := UDT{} + testsCount := 1000 + for i := 0; i < testsCount; i++ { + elems := 20 + maxElemLen := 10 + if i == 0 { + elems = 0 + maxElemLen = 0 + } + expected, data := rndDataElems(elems, maxElemLen, false, false) + // UDT initialization. + testColumn.Names = make([]string, elems) + testColumn.Values = make([]Elem, elems) + for idx := range testColumn.Values { + testColumn.Values[idx] = &ColumnRaw{} + } + // Unmarshall. + err := testColumn.UnmarshalCQL(gocql.UDTTypeInfo{Elements: make([]gocql.UDTField, elems)}, data) + // Check results. + if err != nil { + t.Fatalf("%s error:%s", errorMsg, err) + } + if len(testColumn.Values) != len(expected) { + t.Fatalf("%s\nreceived len:%d \nexpected len:%d", errorMsg, len(testColumn.Values), len(expected)) + } + for idx := range testColumn.Values { + if !reflect.DeepEqual(expected[idx], testColumn.Values[idx]) { + t.Fatalf("%s\nreceived:%+v \nexpected:%+v", errorMsg, testColumn.Values[idx], expected[idx]) + } + } + testColumn.Names = make([]string, 0) + testColumn.Values = make([]Elem, 0) + } +} + +func TestUDT_Equal(t *testing.T) { + var testColumn1, testColumn2 UDT + for i := range testCeases { + testColumn1.Names, testColumn2.Names = rndSameStrings(testCeases[i].elems, testCeases[i].elemLen) + testColumn1.Values, testColumn2.Values = rndSameElems(testCeases[i].elems, testCeases[i].elemLen) + // EqualColumn test on equal + if !testColumn1.EqualColumn(testColumn2) { + t.Fatal("UDT.EqualColumn should return true") + } + // EqualElem test on equal + if !testColumn1.EqualElem(&testColumn2) { + t.Fatal("UDT.EqualElem should return true") + } + // Corrupt values + testColumn2.Values = []Elem{ + &ColumnRaw{1, 2, 3}, + } + // EqualColumn test on unequal + if testColumn1.EqualColumn(testColumn2) { + t.Fatal("UDT.EqualColumn should return false") + } + // EqualElem test on unequal + if testColumn1.EqualElem(&testColumn2) { + t.Fatal("UDT.EqualElem should return false") + } + + // Corrupt names + testColumn1.Values, testColumn2.Values = rndSameElems(testCeases[i].elems, testCeases[i].elemLen) + testColumn2.Names = []string{ + "corrupt names", + } + // EqualColumn test on unequal + if testColumn1.EqualColumn(testColumn2) { + t.Fatal("UDT.EqualColumn should return false") + } + // EqualElem test on unequal + if testColumn1.EqualElem(&testColumn2) { + t.Fatal("UDT.EqualElem should return false") + } + } +} diff --git a/pkg/store/mv/delete_equal_test.go b/pkg/store/mv/delete_equal_test.go new file mode 100644 index 00000000..11475806 --- /dev/null +++ b/pkg/store/mv/delete_equal_test.go @@ -0,0 +1,108 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package mv + +import ( + "reflect" + "testing" +) + +func TestEasyDeleteEqualRows(t *testing.T) { + var test Results + test.Oracle.Rows, test.Test.Rows = rndSameRows(20, 20, 20) + test.Test.Rows[10] = rndRow(20, 20) + + expectedOracle := test.Oracle.Rows[10:] + expectedTest := test.Test.Rows[10:] + // Test EasyDeleteEqualRows with middle unequal row + runEasyDeleteEqualRowsTest(t, &test, expectedOracle, expectedTest, 10) + + // Test EasyDeleteEqualRows with fist unequal row + runEasyDeleteEqualRowsTest(t, &test, test.Oracle.Rows, test.Test.Rows, 0) + + test.Test.Rows[0] = test.Oracle.Rows[0] + test.Oracle.Rows[9] = rndRow(20, 20) + expectedOracle = test.Oracle.Rows[9:] + expectedTest = test.Test.Rows[9:] + + // Test EasyDeleteEqualRows with last unequal row + runEasyDeleteEqualRowsTest(t, &test, expectedOracle, expectedTest, 9) + + test.Test.Rows[0] = test.Oracle.Rows[0] + + // Test EasyDeleteEqualRows with one equal row + runEasyDeleteEqualRowsTest(t, &test, RowsMV{}, RowsMV{}, 1) + + // Test EasyDeleteEqualRows with nil rows + runEasyDeleteEqualRowsTest(t, &test, RowsMV{}, RowsMV{}, 0) +} + +func TestDeleteEqualRows(t *testing.T) { + var test Results + test.Oracle.Rows, test.Test.Rows = rndSameRows(20, 20, 20) + test.Test.Rows[10] = rndRow(20, 20) + test.Test.Rows[11] = rndRow(20, 20) + test.Test.Rows[12] = rndRow(20, 20) + + expectedOracle := RowsMV{test.Oracle.Rows[12], test.Oracle.Rows[11], test.Oracle.Rows[10]} + expectedTest := RowsMV{test.Test.Rows[12], test.Test.Rows[11], test.Test.Rows[10]} + // Test DeleteEqualRows with middle unequal row + runDeleteEqualRowsTest(t, &test, expectedOracle, expectedTest, 17) + + test.Test.Rows[1] = test.Oracle.Rows[1] + test.Test.Rows[2] = test.Oracle.Rows[2] + expectedOracle = RowsMV{test.Oracle.Rows[0]} + expectedTest = RowsMV{test.Test.Rows[0]} + + // Test DeleteEqualRows with fist unequal row + runDeleteEqualRowsTest(t, &test, expectedOracle, expectedTest, 2) + + test.Test.Rows[0] = test.Oracle.Rows[0] + // Test DeleteEqualRows with one equal row + runDeleteEqualRowsTest(t, &test, RowsMV{}, RowsMV{}, 1) + + // Test DeleteEqualRows with nil rows + runDeleteEqualRowsTest(t, &test, RowsMV{}, RowsMV{}, 0) +} + +func runEasyDeleteEqualRowsTest(t *testing.T, test *Results, expectedOracle, expectedTest RowsMV, expectedDeletes int) { + t.Helper() + deleteCount := test.EasyEqualRowsTest() + if !reflect.DeepEqual(test.Oracle.Rows, expectedOracle) { + t.Fatalf("wrong EasyDeleteEqualRows work. \nreceived:%+v \nexpected:%+v", test.Oracle.Rows, expectedOracle) + } + if !reflect.DeepEqual(test.Test.Rows, expectedTest) { + t.Fatalf("wrong EasyDeleteEqualRows work. \nreceived:%+v \nexpected:%+v", test.Test.Rows, expectedTest) + } + if deleteCount != expectedDeletes { + t.Fatalf("wrong EasyDeleteEqualRows work. deletes count %d, but should %d ", deleteCount, expectedDeletes) + } +} + +func runDeleteEqualRowsTest(t *testing.T, test *Results, expectedOracle, expectedTest RowsMV, expectedDeletes int) { + t.Helper() + deleteCount := test.EqualRowsTest() + if !reflect.DeepEqual(test.Oracle.Rows, expectedOracle) { + t.Fatalf("wrong DeleteEqualRows work. \nreceived:%+v \nexpected:%+v", test.Oracle.Rows, expectedOracle) + } + if !reflect.DeepEqual(test.Test.Rows, expectedTest) { + t.Fatalf("wrong DeleteEqualRows work. \nreceived:%+v \nexpected:%+v", test.Test.Rows, expectedTest) + } + if deleteCount != expectedDeletes { + t.Fatalf("wrong DeleteEqualRows work. deletes count %d, but should %d ", deleteCount, expectedDeletes) + } +} diff --git a/pkg/store/mv/init_column.go b/pkg/store/mv/init_column.go new file mode 100644 index 00000000..87ff8473 --- /dev/null +++ b/pkg/store/mv/init_column.go @@ -0,0 +1,126 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +import "github.com/gocql/gocql" + +// initColumn returns Column implementation for specified 'typeInfo'. +// CQL protocols <=2 and >2 have different len of the for collection types (List, Set and Map). +// The simplest type representation - ColumnRaw{}, but it can be used only for not collection types and for Tuple, UDT types without collection types. +// All collection types and Tuple, UDT types with collection types have own representations. +// So initColumn checks all these cases and returns right Column implementation. +func initColumn(colType gocql.TypeInfo) Column { + if !haveCollection(colType) { + return ColumnRaw{} + } + + switch colType.Type() { + case gocql.TypeList, gocql.TypeSet: + listType := colType.(gocql.CollectionType) + return List{ + initElem(listType.Elem), + } + case gocql.TypeMap: + mapType := colType.(gocql.CollectionType) + return Map{ + Keys: []Elem{initElem(mapType.Key)}, + Values: []Elem{initElem(mapType.Elem)}, + } + case gocql.TypeTuple: + elems := colType.(gocql.TupleTypeInfo).Elems + tuple := make(Tuple, len(elems)) + for i, elem := range elems { + tuple[i] = initElem(elem) + } + return tuple + case gocql.TypeUDT: + elems := colType.(gocql.UDTTypeInfo).Elements + udt := UDT{ + Names: make([]string, len(elems)), + Values: make([]Elem, len(elems)), + } + for i := range elems { + udt.Names[i] = elems[i].Name + udt.Values[i] = initElem(elems[i].Type) + } + return udt + default: + return ColumnRaw{} + } +} + +// haveCollection returns true if the collection type is present in the specified type. +func haveCollection(typeInfo gocql.TypeInfo) bool { + switch typeInfo.Type() { + case gocql.TypeList, gocql.TypeSet, gocql.TypeMap: + return true + case gocql.TypeUDT: + udt := typeInfo.(gocql.UDTTypeInfo) + for idx := range udt.Elements { + if haveCollection(udt.Elements[idx].Type) { + return true + } + } + case gocql.TypeTuple: + tuple := typeInfo.(gocql.TupleTypeInfo) + for idx := range tuple.Elems { + if haveCollection(tuple.Elems[idx]) { + return true + } + } + } + return false +} + +// initElem returns Elem implementation for the specified type. +func initElem(elemType gocql.TypeInfo) Elem { + if !haveCollection(elemType) { + return &ColumnRaw{} + } + + switch elemType.Type() { + case gocql.TypeList, gocql.TypeSet: + listType := elemType.(gocql.CollectionType) + return &List{ + initElem(listType.Elem), + } + case gocql.TypeMap: + mapType := elemType.(gocql.CollectionType) + return &Map{ + Keys: []Elem{initElem(mapType.Key)}, + Values: []Elem{initElem(mapType.Elem)}, + } + case gocql.TypeTuple: + elems := elemType.(gocql.TupleTypeInfo).Elems + tuple := make(Tuple, len(elems)) + for i, elem := range elems { + tuple[i] = initElem(elem) + } + return &tuple + case gocql.TypeUDT: + elems := elemType.(gocql.UDTTypeInfo).Elements + udt := UDT{ + Names: make([]string, len(elems)), + Values: make([]Elem, len(elems)), + } + for i := range elems { + udt.Names[i] = elems[i].Name + udt.Values[i] = initElem(elems[i].Type) + } + return &udt + default: + return &ColumnRaw{} + } +} diff --git a/pkg/store/mv/interface.go b/pkg/store/mv/interface.go new file mode 100644 index 00000000..5bb77fbf --- /dev/null +++ b/pkg/store/mv/interface.go @@ -0,0 +1,42 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +import ( + "github.com/gocql/gocql" +) + +// Column is an interface that describes the necessary methods for the processing with columns. +// To unmarshall by the iter.Scan function need to put references of Column's implementations in interface{}. +// So we can put Column's implementations (not references) in the RowMV ([]Column) and send Column's references to the iter.Scan function by interface{}. +type Column interface { + // NewSameColumn creates the same but empty column. + NewSameColumn() Column + EqualColumn(interface{}) bool + // ToString makes string from Column's with interpretations into GO types. + ToString(gocql.TypeInfo) string + // ToStringRaw makes string from Column's without interpretations into GO types. + ToStringRaw() string + // ToUnmarshal makes references Column's implementations in the interface{}. Necessary to unmarshall by the iter.Scan function. + ToUnmarshal() interface{} +} + +// Elem is an interface that describes the necessary methods for the processing with elements of the collection, Tuple, UDT types. +type Elem interface { + Column + NewSameElem() Elem + EqualElem(interface{}) bool + UnmarshalCQL(colInfo gocql.TypeInfo, data []byte) error +} diff --git a/pkg/store/mv/result.go b/pkg/store/mv/result.go new file mode 100644 index 00000000..3011bd42 --- /dev/null +++ b/pkg/store/mv/result.go @@ -0,0 +1,126 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +import ( + "fmt" + + "github.com/gocql/gocql" + + "github.com/scylladb/gemini/pkg/store/ver" +) + +// GetResult scan gocql.Iter and returns Result. +func GetResult(iter *gocql.Iter) Result { + switch iter.NumRows() { + case 0: + return Result{} + case 1: + out := initResult(iter) + out.Rows = initRows(out.Types, iter.NumRows()) + iter.Scan(out.Rows[0].ToUnmarshal()...) + return out + default: + out := initResult(iter) + out.Rows = initRows(out.Types, iter.NumRows()) + count := 0 + for iter.Scan(out.Rows[count].ToUnmarshal()...) { + count++ + if count >= len(out.Rows) { + out.Rows = append(out.Rows, make(RowsMV, len(out.Rows))...) + } + out.Rows[count] = out.Rows[0].NewSameRow() + } + out.Rows = out.Rows[:count] + return out + } +} + +// GetResultWithVerCheck same as GetResult with check of responses on difference in protocol versions. +func GetResultWithVerCheck(iter *gocql.Iter) Result { + switch iter.NumRows() { + case 0: + return Result{} + case 1: + out := initResult(iter) + if !ver.Check.Done() { + ver.Check.Add(out.Types[0].Version() <= 2) + } + out.Rows = initRows(out.Types, iter.NumRows()) + iter.Scan(out.Rows[0].ToUnmarshal()...) + return out + default: + out := initResult(iter) + if !ver.Check.Done() { + ver.Check.Add(out.Types[0].Version() <= 2) + } + out.Rows = initRows(out.Types, iter.NumRows()) + count := 0 + for iter.Scan(out.Rows[count].ToUnmarshal()...) { + count++ + if count >= len(out.Rows) { + out.Rows = append(out.Rows, make(RowsMV, len(out.Rows))...) + } + out.Rows[count] = out.Rows[0].NewSameRow() + } + out.Rows = out.Rows[:count] + return out + } +} + +type Result struct { + Types []gocql.TypeInfo + Names []string + Rows RowsMV +} + +// initResult returns Result with filled Types and Names and initiated Rows. +// Only Rows[0] have proper all column's initiation. +func initResult(iter *gocql.Iter) Result { + out := Result{} + out.Types = make([]gocql.TypeInfo, len(iter.Columns())) + out.Names = make([]string, len(iter.Columns())) + idx := 0 + for _, column := range iter.Columns() { + if col, ok := column.TypeInfo.(gocql.TupleTypeInfo); ok { + tmpTypes := make([]gocql.TypeInfo, len(col.Elems)-1) + tmpNames := make([]string, len(col.Elems)-1) + out.Types = append(out.Types, tmpTypes...) + out.Names = append(out.Names, tmpNames...) + for i := range col.Elems { + out.Types[idx] = col.Elems[i] + out.Names[idx] = fmt.Sprintf("%s.t[%d]", column.Name, i) + idx++ + } + } else { + out.Types[idx] = column.TypeInfo + out.Names[idx] = column.Name + idx++ + } + } + return out +} + +func (d Result) LenColumns() int { + return len(d.Names) +} + +func (d Result) LenRows() int { + return len(d.Rows) +} + +func (d Result) RowsToStrings() []string { + return d.Rows.StringsRows(d.Types, d.Names) +} diff --git a/pkg/store/mv/results.go b/pkg/store/mv/results.go new file mode 100644 index 00000000..0e68c4fa --- /dev/null +++ b/pkg/store/mv/results.go @@ -0,0 +1,125 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +type Results struct { + Oracle Result + Test Result +} + +func (d *Results) HaveRows() bool { + return len(d.Test.Rows) != 0 || len(d.Oracle.Rows) != 0 +} + +func (d *Results) LenRowsTest() int { + return len(d.Test.Rows) +} + +func (d *Results) LenRowsOracle() int { + return len(d.Oracle.Rows) +} + +func (d *Results) StringAllRows(prefix string) []string { + var out []string + if d.Test.LenRows() != 0 { + out = append(out, prefix+" test store rows:") + out = append(out, d.Test.RowsToStrings()...) + } + if d.Oracle.LenRows() != 0 { + out = append(out, prefix+" oracle store rows:") + out = append(out, d.Oracle.RowsToStrings()...) + } + return out +} + +func (d *Results) EqualSingeRow() int { + if d.Test.Rows[0].Equal(d.Oracle.Rows[0]) { + d.Test.Rows = nil + d.Oracle.Rows = nil + return 1 + } + return 0 +} + +func (d *Results) EasyEqualRowsTest() int { + var idx int + // Travel through rows to find fist unequal row. + for range d.Test.Rows { + if d.Test.Rows[idx].Equal(d.Oracle.Rows[idx]) { + idx++ + continue + } + break + } + d.Test.Rows = d.Test.Rows[idx:] + d.Oracle.Rows = d.Oracle.Rows[idx:] + return idx +} + +func (d *Results) EasyEqualRowsOracle() int { + var idx int + // Travel through rows to find fist unequal row. + for range d.Oracle.Rows { + if d.Oracle.Rows[idx].Equal(d.Test.Rows[idx]) { + idx++ + continue + } + break + } + d.Test.Rows = d.Test.Rows[idx:] + d.Oracle.Rows = d.Oracle.Rows[idx:] + return idx +} + +func (d *Results) EqualRowsTest() int { + var equalRowsCount int + idxT := 0 + for range d.Test.Rows { + idxO := d.Oracle.Rows.FindEqualRow(d.Test.Rows[idxT]) + if idxO < 0 { + // No equal row founded - switch to next Test row + idxT++ + continue + } + // EqualColumn row founded - delete equal row from Test and Oracle stores, add equal rows counter + d.Oracle.Rows[idxO] = d.Oracle.Rows[d.Oracle.LenRows()-1] + d.Oracle.Rows = d.Oracle.Rows[:d.Oracle.LenRows()-1] + d.Test.Rows[idxT] = d.Test.Rows[d.Test.LenRows()-1] + d.Test.Rows = d.Test.Rows[:d.Test.LenRows()-1] + equalRowsCount++ + } + return equalRowsCount +} + +func (d *Results) EqualRowsOracle() int { + var equalRowsCount int + // Travel through the fewer rows + idxO := 0 + for range d.Oracle.Rows { + idxT := d.Test.Rows.FindEqualRow(d.Oracle.Rows[idxO]) + if idxT < 0 { + // No equal row founded - switch to next Oracle row + idxO++ + continue + } + // EqualColumn row founded - delete equal row from Test and Oracle stores, add equal rows counter + d.Test.Rows[idxT] = d.Test.Rows[d.Test.LenRows()-1] + d.Test.Rows = d.Test.Rows[:d.Test.LenRows()-1] + d.Oracle.Rows[idxO] = d.Oracle.Rows[d.Oracle.LenRows()-1] + d.Oracle.Rows = d.Oracle.Rows[:d.Oracle.LenRows()-1] + equalRowsCount++ + } + return equalRowsCount +} diff --git a/pkg/store/mv/row.go b/pkg/store/mv/row.go new file mode 100644 index 00000000..d2b79ac2 --- /dev/null +++ b/pkg/store/mv/row.go @@ -0,0 +1,69 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +import ( + "fmt" + + "github.com/gocql/gocql" +) + +type RowMV []Column + +func (row RowMV) Len() int { + return len(row) +} + +func (row RowMV) String(types []gocql.TypeInfo, names []string) string { + out := "" + if len(row) == 0 { + return out + } + for idx := range row { + out += fmt.Sprintf("%s:%+v", names[idx], row[idx].ToString(types[idx])) + columnsSeparator + } + return out[:len(out)-1] +} + +func (row RowMV) Equal(row2 RowMV) bool { + if len(row) != len(row2) { + return false + } + if len(row) == 0 { + return true + } + for idx := range row { + if !row2[idx].EqualColumn(row[idx]) { + return false + } + } + return true +} + +func (row RowMV) ToUnmarshal() []interface{} { + out := make([]interface{}, len(row)) + for idx := range row { + out[idx] = row[idx].ToUnmarshal() + } + return out +} + +func (row RowMV) NewSameRow() RowMV { + out := make(RowMV, len(row)) + for idx := range row { + out[idx] = row[idx].NewSameColumn() + } + return out +} diff --git a/pkg/store/mv/rows.go b/pkg/store/mv/rows.go new file mode 100644 index 00000000..326b44f2 --- /dev/null +++ b/pkg/store/mv/rows.go @@ -0,0 +1,56 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +import ( + "fmt" + + "github.com/gocql/gocql" +) + +const columnsSeparator = ";" + +func initRows(types []gocql.TypeInfo, rows int) RowsMV { + out := make(RowsMV, rows) + baseRow := make(RowMV, len(types)) + for idx := range types { + baseRow[idx] = initColumn(types[idx]) + } + out[0] = baseRow + return out +} + +type RowsMV []RowMV + +func (l RowsMV) LenRows() int { + return len(l) +} + +func (l RowsMV) StringsRows(types []gocql.TypeInfo, names []string) []string { + out := make([]string, len(l)) + for idx := range l { + out[idx] = fmt.Sprintf("row%d:%s", idx, l[idx].String(types, names)) + } + return out +} + +func (l RowsMV) FindEqualRow(row RowMV) int { + for idx := range l { + if row.Equal(l[idx]) { + return idx + } + } + return -1 +} diff --git a/pkg/store/mv/utils.go b/pkg/store/mv/utils.go new file mode 100644 index 00000000..2327715c --- /dev/null +++ b/pkg/store/mv/utils.go @@ -0,0 +1,38 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mv + +import ( + "reflect" + "unsafe" + + "github.com/pkg/errors" +) + +func dereference(in interface{}) interface{} { + return reflect.Indirect(reflect.ValueOf(in)).Interface() +} + +func readLen(p []byte) int32 { + return *(*int32)(unsafe.Pointer(&p[0])) +} + +func readLenOld(p []byte) int16 { + return *(*int16)(unsafe.Pointer(&p[0])) +} + +const protoVersion2 = 0x02 + +var ErrorUnmarshalEOF = errors.New("unexpected eof") diff --git a/pkg/store/mv/utils_4test.go b/pkg/store/mv/utils_4test.go new file mode 100644 index 00000000..852ab976 --- /dev/null +++ b/pkg/store/mv/utils_4test.go @@ -0,0 +1,189 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package mv + +import ( + "time" + "unsafe" + + "golang.org/x/exp/rand" + + "github.com/scylladb/gemini/pkg/utils" +) + +var rnd = rand.New(rand.NewSource(uint64(time.Now().UnixMilli()))) + +type testCase struct { + elems, elemLen int +} + +var testCeases = []testCase{ + {0, 0}, + {1, 0}, + {1, 1}, + {1, 10}, + {10, 10}, + {10, 0}, +} + +func rndDataElems(elems, elemLenMax int, old, collection bool) ([]Elem, []byte) { + out1 := make([]Elem, elems) + out2 := make([]byte, 0) + elemLen := 0 + elemLenArr := unsafe.Slice((*byte)(unsafe.Pointer(&elemLen)), 4) + if old { + elemLenArr = unsafe.Slice((*byte)(unsafe.Pointer(&elemLen)), 2) + } + if collection { + elemLen = elems + out2 = append(out2, elemLenArr...) + } + for idx := range out1 { + elemLen = rand.Intn(elemLenMax) + rndData := ColumnRaw(utils.RandBytes(rnd, elemLen)) + out2 = append(out2, elemLenArr...) + out2 = append(out2, rndData...) + out1[idx] = &rndData + } + return out1, out2 +} + +func rndDataElemsMap(elems, elemLenMax int, old bool) ([]Elem, []Elem, []byte) { + outKeys := make([]Elem, elems) + outValues := make([]Elem, elems) + outData := make([]byte, 0) + elemLen := elems + elemLenArr := unsafe.Slice((*byte)(unsafe.Pointer(&elemLen)), 4) + if old { + elemLenArr = unsafe.Slice((*byte)(unsafe.Pointer(&elemLen)), 2) + } + outData = append(outData, elemLenArr...) + for idx := range outKeys { + // Add key + elemLen = rand.Intn(elemLenMax) + rndKeyData := ColumnRaw(utils.RandBytes(rnd, elemLen)) + outData = append(outData, elemLenArr...) + outData = append(outData, rndKeyData...) + outKeys[idx] = &rndKeyData + // Add value + elemLen = rand.Intn(elemLenMax) + rndValueData := ColumnRaw(utils.RandBytes(rnd, elemLen)) + outData = append(outData, elemLenArr...) + outData = append(outData, rndValueData...) + outValues[idx] = &rndValueData + } + if elems < 1 { + // With correction needed, because List and Map initialization required fist elem. + outKeys = append(outKeys, &ColumnRaw{}) + outValues = append(outValues, &ColumnRaw{}) + } + return outKeys, outValues, outData +} + +func rndRow(columns, columnLen int) RowMV { + out := make(RowMV, columns) + for idx := range out { + col := utils.RandBytes(rnd, columnLen) + out[idx] = *(*ColumnRaw)(&col) + } + return out +} + +func rndSameRows(rowsCount, columns, columnLen int) (RowsMV, RowsMV) { + out1 := make(RowsMV, rowsCount) + out2 := make(RowsMV, rowsCount) + for idx := range out1 { + out1[idx], out2[idx] = rndSameRow(columns, columnLen) + } + return out1, out2 +} + +func rndSameRow(columns, columnLen int) (RowMV, RowMV) { + out1 := make(RowMV, columns) + out2 := make(RowMV, columns) + for id := 0; id < columns; id++ { + switch id % 5 { + case 0: + out1[id], out2[id] = rndSameRaw(columnLen) + case 1: + out1[id], out2[id] = rndSameLists(2, columnLen) + case 2: + out1[id], out2[id] = rndSameMaps(2, columnLen) + case 3: + out1[id], out2[id] = rndSameTuples(2, columnLen) + case 4: + out1[id], out2[id] = rndSameUDTs(2, columnLen) + } + } + return out1, out2 +} + +func rndSameRaw(columnLen int) (ColumnRaw, ColumnRaw) { + out1 := make(ColumnRaw, columnLen) + for idx := range out1 { + out1[idx] = byte(rnd.Intn(256)) + } + out2 := make(ColumnRaw, columnLen) + copy(out2, out1) + return out1, out2 +} + +func rndSameStrings(elems, elemLen int) ([]string, []string) { + out1 := make([]string, elems) + for idx := range out1 { + out1[idx] = utils.RandString(rnd, elemLen) + } + out2 := make([]string, elems) + copy(out2, out1) + return out1, out2 +} + +func rndSameElems(elems, elemLen int) ([]Elem, []Elem) { + out1 := make([]Elem, elems) + out2 := make([]Elem, elems) + for id := range out1 { + tmp1, tmp2 := rndSameRaw(elemLen) + out1[id], out2[id] = &tmp1, &tmp2 + } + return out1, out2 +} + +func rndSameLists(elems, elemLen int) (List, List) { + var out1, out2 List + out1, out2 = rndSameElems(elems, elemLen) + return out1, out2 +} + +func rndSameMaps(elems, elemLen int) (Map, Map) { + var out1, out2 Map + out1.Keys, out2.Keys = rndSameElems(elems, elemLen) + out1.Values, out2.Values = rndSameElems(elems, elemLen) + return out1, out2 +} + +func rndSameTuples(elems, elemLen int) (Tuple, Tuple) { + var out1, out2 Tuple + out1, out2 = rndSameElems(elems, elemLen) + return out1, out2 +} + +func rndSameUDTs(elems, elemLen int) (UDT, UDT) { + var out1, out2 UDT + out1.Names, out2.Names = rndSameStrings(elems, elemLen) + out1.Values, out2.Values = rndSameElems(elems, elemLen) + return out1, out2 +} diff --git a/pkg/store/store.go b/pkg/store/store.go index ed5fd98a..ed945ba8 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -16,32 +16,31 @@ package store import ( "context" - "fmt" - "math/big" "os" - "reflect" - "sort" "sync" "time" - "go.uber.org/zap" - "github.com/gocql/gocql" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/scylladb/go-set/strset" "github.com/scylladb/gocqlx/v2/qb" "go.uber.org/multierr" - "gopkg.in/inf.v0" + "go.uber.org/zap" + "github.com/scylladb/gemini/pkg/store/comp" + mv "github.com/scylladb/gemini/pkg/store/mv" + sv "github.com/scylladb/gemini/pkg/store/sv" + "github.com/scylladb/gemini/pkg/store/ver" "github.com/scylladb/gemini/pkg/typedef" ) +var errorResponseDiffer = errors.New("response from test and oracle store have difference") + type loader interface { - load(context.Context, qb.Builder, []interface{}) ([]map[string]interface{}, error) + loadSV(context.Context, qb.Builder, []interface{}) (sv.Result, error) + loadMV(context.Context, qb.Builder, []interface{}) (mv.Result, error) + loadVerCheck(context.Context, qb.Builder, []interface{}) (mv.Result, error) } type storer interface { @@ -129,8 +128,16 @@ func (n *noOpStore) mutate(context.Context, qb.Builder, ...interface{}) error { return nil } -func (n *noOpStore) load(context.Context, qb.Builder, []interface{}) ([]map[string]interface{}, error) { - return nil, nil +func (n *noOpStore) loadSV(context.Context, qb.Builder, []interface{}) (sv.Result, error) { + return sv.Result{}, nil +} + +func (n *noOpStore) loadMV(context.Context, qb.Builder, []interface{}) (mv.Result, error) { + return mv.Result{}, nil +} + +func (n *noOpStore) loadVerCheck(context.Context, qb.Builder, []interface{}) (mv.Result, error) { + return mv.Result{}, nil } func (n *noOpStore) Close() error { @@ -191,16 +198,37 @@ func mutate(ctx context.Context, s storeLoader, builder qb.Builder, values ...in return nil } -func (ds delegatingStore) Check(ctx context.Context, table *typedef.Table, builder qb.Builder, detailedDiff bool, values ...interface{}) error { - var testRows, oracleRows []map[string]interface{} +func (ds delegatingStore) Check(ctx context.Context, _ *typedef.Table, builder qb.Builder, detailedDiff bool, values ...interface{}) error { var testErr, oracleErr error var wg sync.WaitGroup wg.Add(1) - go func() { - testRows, testErr = ds.testStore.load(ctx, builder, values) - wg.Done() - }() - oracleRows, oracleErr = ds.oracleStore.load(ctx, builder, values) + var results comp.Results + switch { + case ver.Check.ModeSV(): + resultsSV := sv.Results{} + go func() { + resultsSV.Test, testErr = ds.testStore.loadSV(ctx, builder, values) + wg.Done() + }() + resultsSV.Oracle, oracleErr = ds.oracleStore.loadSV(ctx, builder, values) + results = &resultsSV + case ver.Check.Done(): + resultsMV := mv.Results{} + go func() { + resultsMV.Test, testErr = ds.testStore.loadMV(ctx, builder, values) + wg.Done() + }() + resultsMV.Oracle, oracleErr = ds.oracleStore.loadMV(ctx, builder, values) + results = &resultsMV + default: + resultsMV := mv.Results{} + go func() { + resultsMV.Test, testErr = ds.testStore.loadVerCheck(ctx, builder, values) + wg.Done() + }() + resultsMV.Oracle, oracleErr = ds.oracleStore.loadVerCheck(ctx, builder, values) + results = &resultsMV + } if oracleErr != nil { return errors.Wrapf(oracleErr, "unable to load check data from the oracle store") } @@ -208,50 +236,17 @@ func (ds delegatingStore) Check(ctx context.Context, table *typedef.Table, build if testErr != nil { return errors.Wrapf(testErr, "unable to load check data from the test store") } - if !ds.validations { - return nil - } - if len(testRows) == 0 && len(oracleRows) == 0 { - return nil - } - if len(testRows) != len(oracleRows) { - if !detailedDiff { - return fmt.Errorf("rows count differ (test store rows %d, oracle store rows %d, detailed information will be at last attempt)", len(testRows), len(oracleRows)) - } - testSet := strset.New(pks(table, testRows)...) - oracleSet := strset.New(pks(table, oracleRows)...) - missingInTest := strset.Difference(oracleSet, testSet).List() - missingInOracle := strset.Difference(testSet, oracleSet).List() - return fmt.Errorf("row count differ (test has %d rows, oracle has %d rows, test is missing rows: %s, oracle is missing rows: %s)", - len(testRows), len(oracleRows), missingInTest, missingInOracle) - } - if reflect.DeepEqual(testRows, oracleRows) { + if !ds.validations || !results.HaveRows() { return nil } - if !detailedDiff { - return fmt.Errorf("test and oracle store have difference, detailed information will be at last attempt") + var diff comp.Info + if detailedDiff { + diff = comp.GetCompareInfoDetailed(results) + } else { + diff = comp.GetCompareInfoSimple(results) } - sort.SliceStable(testRows, func(i, j int) bool { - return lt(testRows[i], testRows[j]) - }) - sort.SliceStable(oracleRows, func(i, j int) bool { - return lt(oracleRows[i], oracleRows[j]) - }) - for i, oracleRow := range oracleRows { - testRow := testRows[i] - cmp.AllowUnexported() - diff := cmp.Diff(oracleRow, testRow, - cmpopts.SortMaps(func(x, y *inf.Dec) bool { - return x.Cmp(y) < 0 - }), - cmp.Comparer(func(x, y *inf.Dec) bool { - return x.Cmp(y) == 0 - }), cmp.Comparer(func(x, y *big.Int) bool { - return x.Cmp(y) == 0 - })) - if diff != "" { - return fmt.Errorf("rows differ (-%v +%v): %v", oracleRow, testRow, diff) - } + if diff.Len() > 0 { + return errors.Wrap(errorResponseDiffer, diff.String()) } return nil } diff --git a/pkg/store/sv/delete_equal_test.go b/pkg/store/sv/delete_equal_test.go new file mode 100644 index 00000000..f5db7e8e --- /dev/null +++ b/pkg/store/sv/delete_equal_test.go @@ -0,0 +1,139 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package sv + +import ( + "reflect" + "testing" + "time" + + "golang.org/x/exp/rand" + + "github.com/scylladb/gemini/pkg/utils" +) + +var rnd = rand.New(rand.NewSource(uint64(time.Now().UnixMilli()))) + +func TestEasyDeleteEqualRows(t *testing.T) { + test := Results{ + Oracle: Result{Rows: getRandomRawRows(20, 20, 20)}, + } + test.Test.Rows = make(RowsSV, 20) + copy(test.Test.Rows, test.Oracle.Rows) + test.Test.Rows[10] = rndRow(20, 20) + + expectedOracle := test.Oracle.Rows[10:] + expectedTest := test.Test.Rows[10:] + // Test EasyDeleteEqualRows with middle unequal row + runEasyDeleteEqualRowsAndCheck(t, &test, expectedOracle, expectedTest, 10) + + // Test EasyDeleteEqualRows with fist unequal row + runEasyDeleteEqualRowsAndCheck(t, &test, test.Oracle.Rows, test.Test.Rows, 0) + + test.Test.Rows[0] = test.Oracle.Rows[0] + test.Oracle.Rows[9] = rndRow(20, 20) + expectedOracle = test.Oracle.Rows[9:] + expectedTest = test.Test.Rows[9:] + + // Test EasyDeleteEqualRows with last unequal row + runEasyDeleteEqualRowsAndCheck(t, &test, expectedOracle, expectedTest, 9) + + test.Test.Rows[0] = test.Oracle.Rows[0] + + // Test EasyDeleteEqualRows with one equal row + runEasyDeleteEqualRowsAndCheck(t, &test, RowsSV{}, RowsSV{}, 1) + + // Test EasyDeleteEqualRows with nil rows + runEasyDeleteEqualRowsAndCheck(t, &test, RowsSV{}, RowsSV{}, 0) +} + +func runEasyDeleteEqualRowsAndCheck(t *testing.T, test *Results, expectedOracle, expectedTest RowsSV, expectedDeletes int) { + t.Helper() + deleteCount := test.EasyEqualRowsTest() + if !reflect.DeepEqual(test.Oracle.Rows, expectedOracle) { + t.Fatalf("wrong EasyDeleteEqualRows work. \nreceived:%+v \nexpected:%+v", test.Oracle.Rows, expectedOracle) + } + if !reflect.DeepEqual(test.Test.Rows, expectedTest) { + t.Fatalf("wrong EasyDeleteEqualRows work. \nreceived:%+v \nexpected:%+v", test.Test.Rows, expectedTest) + } + if deleteCount != expectedDeletes { + t.Fatalf("wrong EasyDeleteEqualRows work. deletes count %d, but should %d ", deleteCount, expectedDeletes) + } +} + +func TestDeleteEqualRows(t *testing.T) { + test := Results{ + Oracle: Result{Rows: getRandomRawRows(20, 20, 20)}, + } + test.Test.Rows = make(RowsSV, 20) + copy(test.Test.Rows, test.Oracle.Rows) + test.Test.Rows[10] = rndRow(20, 20) + test.Test.Rows[11] = rndRow(20, 20) + test.Test.Rows[12] = rndRow(20, 20) + + expectedOracle := RowsSV{test.Oracle.Rows[12], test.Oracle.Rows[11], test.Oracle.Rows[10]} + expectedTest := RowsSV{test.Test.Rows[12], test.Test.Rows[11], test.Test.Rows[10]} + // Test DeleteEqualRows with middle unequal row + runDeleteEqualRows(t, &test, expectedOracle, expectedTest, 17) + + test.Test.Rows[1] = test.Oracle.Rows[1] + test.Test.Rows[2] = test.Oracle.Rows[2] + expectedOracle = RowsSV{test.Oracle.Rows[0]} + expectedTest = RowsSV{test.Test.Rows[0]} + + // Test DeleteEqualRows with fist unequal row + runDeleteEqualRows(t, &test, expectedOracle, expectedTest, 2) + + test.Test.Rows[0] = test.Oracle.Rows[0] + // Test DeleteEqualRows with one equal row + runDeleteEqualRows(t, &test, RowsSV{}, RowsSV{}, 1) + + // Test DeleteEqualRows with nil rows + runDeleteEqualRows(t, &test, RowsSV{}, RowsSV{}, 0) +} + +func runDeleteEqualRows(t *testing.T, test *Results, expectedOracle, expectedTest RowsSV, expectedDeletes int) { + t.Helper() + deleteCount := test.EqualRowsTest() + if !reflect.DeepEqual(test.Oracle.Rows, expectedOracle) { + t.Fatalf("wrong DeleteEqualRows work. \nreceived:%+v \nexpected:%+v", test.Oracle.Rows, expectedOracle) + } + if !reflect.DeepEqual(test.Test.Rows, expectedTest) { + t.Fatalf("wrong DeleteEqualRows work. \nreceived:%+v \nexpected:%+v", test.Test.Rows, expectedTest) + } + if deleteCount != expectedDeletes { + t.Fatalf("wrong DeleteEqualRows work. deletes count %d, but should %d ", deleteCount, expectedDeletes) + } +} + +func getRandomRawRows(rowsCount, columns, columnLen int) RowsSV { + out := make(RowsSV, rowsCount) + for idx := range out { + tmp := rndRow(columns, columnLen) + out[idx] = tmp + } + return out +} + +func rndRow(columns, columnLen int) RowSV { + out := make(RowSV, columns) + for idx := range out { + col := utils.RandBytes(rnd, columnLen) + out[idx] = *(*ColumnRaw)(&col) + } + return out +} diff --git a/pkg/store/sv/raw.go b/pkg/store/sv/raw.go new file mode 100644 index 00000000..5ac5b055 --- /dev/null +++ b/pkg/store/sv/raw.go @@ -0,0 +1,66 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sv + +import ( + "fmt" + "reflect" + + "github.com/gocql/gocql" +) + +// ColumnRaw for most cases. +type ColumnRaw []byte + +func (col ColumnRaw) ToString(colInfo gocql.TypeInfo) string { + if len(col) == 0 { + return "" + } + tmpVal := colInfo.New() + if err := gocql.Unmarshal(colInfo, col, tmpVal); err != nil { + panic(err) + } + out := fmt.Sprintf("%v", reflect.Indirect(reflect.ValueOf(tmpVal)).Interface()) + // Add name of the type for complex and collections types + switch colInfo.Type() { + case gocql.TypeList: + out = fmt.Sprintf("list<%s>", out) + case gocql.TypeSet: + out = fmt.Sprintf("set<%s>", out) + case gocql.TypeMap: + out = fmt.Sprintf("map<%s>", out) + case gocql.TypeTuple: + out = fmt.Sprintf("tuple<%s>", out) + case gocql.TypeUDT: + out = fmt.Sprintf("udt<%s>", out) + } + return out +} + +func (col ColumnRaw) ToStringRaw() string { + return fmt.Sprint(col) +} + +func (col ColumnRaw) ToInterface() interface{} { + return &col +} + +func (col *ColumnRaw) UnmarshalCQL(_ gocql.TypeInfo, data []byte) error { + if len(data) > 0 { + // Puts data without copying + *col = data + } + return nil +} diff --git a/pkg/store/sv/raw_test.go b/pkg/store/sv/raw_test.go new file mode 100644 index 00000000..331b03df --- /dev/null +++ b/pkg/store/sv/raw_test.go @@ -0,0 +1,42 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package sv + +import ( + "reflect" + "testing" + + "github.com/scylladb/gemini/pkg/utils" +) + +func TestColumnRaw_UnmarshalCQL(t *testing.T) { + errorMsg := "wrong ColumnRaw.UnmarshalCQL work:" + testColumn := make(ColumnRaw, 0) + + testsCount := 1000 + for i := 0; i < testsCount; i++ { + expected := utils.RandBytes(rnd, rnd.Intn(1000)) + if i == 0 { + expected = ColumnRaw{} + } + _ = testColumn.UnmarshalCQL(nil, expected) + if !reflect.DeepEqual(expected, ([]byte)(testColumn)) { + t.Fatalf("%s\nreceived:%+v \nexpected:%+v", errorMsg, testColumn, expected) + } + testColumn = make(ColumnRaw, 0) + } +} diff --git a/pkg/store/sv/result.go b/pkg/store/sv/result.go new file mode 100644 index 00000000..d58f2c06 --- /dev/null +++ b/pkg/store/sv/result.go @@ -0,0 +1,91 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sv + +import ( + "fmt" + + "github.com/gocql/gocql" +) + +// GetResult scan gocql.Iter and returns Result. +func GetResult(iter *gocql.Iter) Result { + switch iter.NumRows() { + case 0: + return Result{} + case 1: + out := initResult(iter) + out.Rows = initRows(len(out.Names), iter.NumRows()) + iter.Scan(out.Rows[0].ToInterfaces()...) + return out + default: + out := initResult(iter) + out.Rows = initRows(len(out.Names), iter.NumRows()) + count := 0 + for iter.Scan(out.Rows[count].ToInterfaces()...) { + count++ + if count >= len(out.Rows) { + out.Rows = append(out.Rows, initRows(len(out.Names), out.LenRows())...) + } + } + out.Rows = out.Rows[:count] + return out + } +} + +type Result struct { + Types []gocql.TypeInfo + Names []string + Rows RowsSV +} + +func (d Result) LenColumns() int { + return len(d.Names) +} + +func (d Result) LenRows() int { + return len(d.Rows) +} + +func (d Result) StringAllRows() []string { + return d.Rows.StringsRows(d.Types, d.Names) +} + +// initResult returns Result with filled Types and Names and initiated Rows. +// Only Rows[0] have proper all column's initiation. +func initResult(iter *gocql.Iter) Result { + out := Result{} + out.Types = make([]gocql.TypeInfo, len(iter.Columns())) + out.Names = make([]string, len(iter.Columns())) + idx := 0 + for _, column := range iter.Columns() { + if col, ok := column.TypeInfo.(gocql.TupleTypeInfo); ok { + tmpTypes := make([]gocql.TypeInfo, len(col.Elems)-1) + tmpNames := make([]string, len(col.Elems)-1) + out.Types = append(out.Types, tmpTypes...) + out.Names = append(out.Names, tmpNames...) + for i := range col.Elems { + out.Types[idx] = col.Elems[i] + out.Names[idx] = fmt.Sprintf("%s.t[%d]", column.Name, i) + idx++ + } + } else { + out.Types[idx] = column.TypeInfo + out.Names[idx] = column.Name + idx++ + } + } + return out +} diff --git a/pkg/store/sv/results.go b/pkg/store/sv/results.go new file mode 100644 index 00000000..3b668804 --- /dev/null +++ b/pkg/store/sv/results.go @@ -0,0 +1,124 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sv + +type Results struct { + Oracle Result + Test Result +} + +func (d *Results) HaveRows() bool { + return len(d.Test.Rows) != 0 || len(d.Oracle.Rows) != 0 +} + +func (d *Results) LenRowsTest() int { + return len(d.Test.Rows) +} + +func (d *Results) LenRowsOracle() int { + return len(d.Oracle.Rows) +} + +func (d *Results) StringAllRows(prefix string) []string { + var out []string + if d.Test.LenRows() != 0 { + out = append(out, prefix+" test store rows:") + out = append(out, d.Test.StringAllRows()...) + } + if d.Oracle.LenRows() != 0 { + out = append(out, prefix+" oracle store rows:") + out = append(out, d.Oracle.StringAllRows()...) + } + return out +} + +func (d *Results) EqualSingeRow() int { + if d.Test.Rows[0].Equal(d.Oracle.Rows[0]) { + d.Test.Rows = nil + d.Oracle.Rows = nil + return 1 + } + return 0 +} + +func (d *Results) EasyEqualRowsTest() int { + var idx int + // Travel through rows to find fist unequal row. + for range d.Test.Rows { + if d.Test.Rows[idx].Equal(d.Oracle.Rows[idx]) { + idx++ + continue + } + break + } + d.Test.Rows = d.Test.Rows[idx:] + d.Oracle.Rows = d.Oracle.Rows[idx:] + return idx +} + +func (d *Results) EasyEqualRowsOracle() int { + var idx int + // Travel through rows to find fist unequal row. + for range d.Oracle.Rows { + if d.Oracle.Rows[idx].Equal(d.Test.Rows[idx]) { + idx++ + continue + } + break + } + d.Test.Rows = d.Test.Rows[idx:] + d.Oracle.Rows = d.Oracle.Rows[idx:] + return idx +} + +func (d *Results) EqualRowsTest() int { + var equalRowsCount int + idxT := 0 + for range d.Test.Rows { + idxO := d.Oracle.Rows.FindEqualRow(d.Test.Rows[idxT]) + if idxO < 0 { + // No equal row founded - switch to next Test row. + idxT++ + continue + } + // EqualColumn row founded - delete equal row from Test and Oracle stores, add equal rows counter + d.Oracle.Rows[idxO] = d.Oracle.Rows[d.Oracle.LenRows()-1] + d.Oracle.Rows = d.Oracle.Rows[:d.Oracle.LenRows()-1] + d.Test.Rows[idxT] = d.Test.Rows[d.Test.LenRows()-1] + d.Test.Rows = d.Test.Rows[:d.Test.LenRows()-1] + equalRowsCount++ + } + return equalRowsCount +} + +func (d *Results) EqualRowsOracle() int { + var equalRowsCount int + idxO := 0 + for range d.Oracle.Rows { + idxT := d.Test.Rows.FindEqualRow(d.Oracle.Rows[idxO]) + if idxT < 0 { + // No equal row founded - switch to next Oracle row + idxO++ + continue + } + // EqualColumn row founded - delete equal row from Test and Oracle stores, add equal rows counter + d.Test.Rows[idxT] = d.Test.Rows[d.Test.LenRows()-1] + d.Test.Rows = d.Test.Rows[:d.Test.LenRows()-1] + d.Oracle.Rows[idxO] = d.Oracle.Rows[d.Oracle.LenRows()-1] + d.Oracle.Rows = d.Oracle.Rows[:d.Oracle.LenRows()-1] + equalRowsCount++ + } + return equalRowsCount +} diff --git a/pkg/store/sv/row.go b/pkg/store/sv/row.go new file mode 100644 index 00000000..f191024e --- /dev/null +++ b/pkg/store/sv/row.go @@ -0,0 +1,74 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sv + +import ( + "fmt" + + "github.com/gocql/gocql" +) + +type RowSV []ColumnRaw + +const columnsSeparator = ";" + +func (row RowSV) Len() int { + return len(row) +} + +func (row RowSV) String(types []gocql.TypeInfo, names []string) string { + out := "" + if len(row) == 0 { + return out + } + for idx := range row { + out += fmt.Sprintf("%s:%+v", names[idx], row[idx].ToString(types[idx])) + columnsSeparator + } + return out[:len(out)-1] +} + +func (row RowSV) Equal(row2 RowSV) bool { + if len(row) != len(row2) { + // If rows len are different - means rows are unequal + return false + } + if len(row) < 1 { + // If rows len are same and len==0 - means rows are equal + // Conditions "<" or ">" works faster that "==" + return true + } + for idx := range row { + if len(row[idx]) != len(row2[idx]) { + return false + } + if len(row[idx]) < 1 { + continue + } + for id := range row[idx] { + if row[idx][id] != row2[idx][id] { + return false + } + } + } + return true +} + +func (row RowSV) ToInterfaces() []interface{} { + out := make([]interface{}, len(row)) + for idx := range row { + out[idx] = row[idx].ToInterface() + } + return out +} diff --git a/pkg/store/sv/rows.go b/pkg/store/sv/rows.go new file mode 100644 index 00000000..6750505c --- /dev/null +++ b/pkg/store/sv/rows.go @@ -0,0 +1,52 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sv + +import ( + "fmt" + + "github.com/gocql/gocql" +) + +func initRows(columnsCount, rows int) RowsSV { + out := make(RowsSV, rows) + for idx := range out { + out[idx] = make(RowSV, columnsCount) + } + return out +} + +type RowsSV []RowSV + +func (l RowsSV) LenRows() int { + return len(l) +} + +func (l RowsSV) StringsRows(types []gocql.TypeInfo, names []string) []string { + out := make([]string, len(l)) + for idx := range l { + out[idx] = fmt.Sprintf("row%d:%s", idx, l[idx].String(types, names)) + } + return out +} + +func (l RowsSV) FindEqualRow(row RowSV) int { + for idx := range l { + if row.Equal(l[idx]) { + return idx + } + } + return -1 +} diff --git a/pkg/store/ver/check.go b/pkg/store/ver/check.go new file mode 100644 index 00000000..c2fc69b1 --- /dev/null +++ b/pkg/store/ver/check.go @@ -0,0 +1,82 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ver + +import "sync/atomic" + +// info represents information about check of responses on difference in protocol versions. +type info struct { + // count of checked responses on difference in protocol versions. + count atomic.Uint32 + // init indicating a status of check's initialization. + init atomic.Bool + // currentOld indicating a version of the first response. + // True - version <=2, False - version >2. + currentOld atomic.Bool + // svMode indicating a mode (mv - multi protocol version or sv - single protocol version). + svMode atomic.Bool + // pass in a value 'true' indicates a check finished. + pass atomic.Bool +} + +// maxResponsesCount represent max count of responses to check on different versions of protocol. +const maxResponsesCount = uint32(200) + +var Check = info{ + count: atomic.Uint32{}, + init: atomic.Bool{}, + currentOld: atomic.Bool{}, + svMode: atomic.Bool{}, + pass: atomic.Bool{}, +} + +func (p *info) reInit() { + p.count = atomic.Uint32{} + p.init = atomic.Bool{} + p.currentOld = atomic.Bool{} + p.svMode = atomic.Bool{} + p.pass = atomic.Bool{} +} + +func (p *info) ModeSV() bool { + return p.svMode.Load() +} + +func (p *info) Done() bool { + return p.pass.Load() +} + +// Add adds one response check. +// oldVersion 'true' means version <=2, 'false' means version >2. +func (p *info) Add(oldVersion bool) { + if !p.init.Load() { + // Check initialization. + p.currentOld.Store(oldVersion) + p.init.Store(true) + return + } + if oldVersion == p.currentOld.Load() { + diff := Check.count.Add(1) + if diff >= maxResponsesCount || !p.pass.Load() { + // Check done + p.svMode.Store(true) + p.pass.Store(true) + } + } else { + // Different versions detected, check done, single version mode off. + p.svMode.Store(false) + p.pass.Store(true) + } +} diff --git a/pkg/store/ver/check_test.go b/pkg/store/ver/check_test.go new file mode 100644 index 00000000..373ac472 --- /dev/null +++ b/pkg/store/ver/check_test.go @@ -0,0 +1,84 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package ver + +import ( + "sync" + "testing" +) + +func TestCheck(t *testing.T) { + t.Parallel() + parallels := 10 + // Test without version difference. + runParallel(parallels, -1, false) + if !Check.Done() { + t.Fatalf("wrong ver.Check work, Check.Done() shoul return 'true'") + } + if !Check.ModeSV() { + t.Fatalf("wrong ver.Check work, Check.ModeSV() shoul return 'true'") + } + + // Test with version difference in the first response. + Check.reInit() + runParallel(parallels, 0, true) + if !Check.Done() { + t.Fatalf("wrong ver.Check work, Check.Done() shoul return 'true'") + } + if Check.ModeSV() { + t.Fatalf("wrong ver.Check work, Check.ModeSV() shoul return 'false'") + } + + // Test with version difference not in the first response. + Check.reInit() + runParallel(parallels, parallels/2, true) + if !Check.Done() { + t.Fatalf("wrong ver.Check work, Check.Done() shoul return 'true'") + } + if Check.ModeSV() { + t.Fatalf("wrong ver.Check work, Check.ModeSV() shoul return 'false'") + } +} + +func runParallel(parallel, addTo int, addOld bool) { + wg := sync.WaitGroup{} + if addOld && addTo < 1 { + Check.Add(true) + } + for i := 0; i < parallel; i++ { + wg.Add(1) + addOld = false + if addTo == i { + addOld = true + } + go func(add bool) { + l := 0 + if add { + Check.Add(true) + } + defer wg.Done() + for { + Check.Add(false) + l++ + if Check.Done() { + return + } + } + }(addOld) + } + wg.Wait() +} diff --git a/pkg/typedef/types.go b/pkg/typedef/types.go index 8ac6f2e6..8f06aeb4 100644 --- a/pkg/typedef/types.go +++ b/pkg/typedef/types.go @@ -118,6 +118,10 @@ var goCQLTypeMap = map[gocql.Type]gocql.TypeInfo{ gocql.TypeCounter: gocql.NewNativeType(GoCQLProtoVersion4, gocql.TypeCounter, ""), } +func GetGoCQLTypeMap() map[gocql.Type]gocql.TypeInfo { + return goCQLTypeMap +} + type MapType struct { ComplexType string `json:"complex_type"` KeyType SimpleType `json:"key_type"` diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index ed7610b0..152c410e 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -103,3 +103,19 @@ func UUIDFromTime(rnd *rand.Rand) string { } return gocql.UUIDFromTime(RandDate(rnd)).String() } + +func RandBytes(rnd *rand.Rand, lenBytes int) []byte { + out := make([]byte, lenBytes) + for idx := range out { + out[idx] = byte(rnd.Intn(256)) + } + return out +} + +func RandSliceBytes(rnd *rand.Rand, count, lenBytes int) [][]byte { + out := make([][]byte, count) + for idx := range out { + out[idx] = RandBytes(rnd, lenBytes) + } + return out +}