From c3e5c7b93f723c59346edf159f2fb05222ef315d Mon Sep 17 00:00:00 2001 From: illia-li Date: Tue, 5 Sep 2023 21:48:01 -0400 Subject: [PATCH] fix(store): fix inf.Dec and big.Int compare changes in check of responses rows procces: *rows data unmarshalling into byte slices/arrays. *rows data compare as byte slices/arrays, without transformation to 'GO' types. *rows diff information now more user friendly and can be customed. Diff collect row by row to string slice and can be transferred anywhere with better readable. --- pkg/response/comp/delete_equal.go | 61 ++++++ pkg/response/comp/delete_equal_test.go | 107 ++++++++++ pkg/response/comp/diff_info.go | 144 +++++++++++++ pkg/response/data.go | 54 +++++ pkg/response/data_test.go | 55 +++++ pkg/response/diffs/helpers.go | 70 ++++++ pkg/response/diffs/info.go | 91 ++++++++ pkg/response/interface.go | 32 +++ pkg/response/protocol/proto_version.go | 70 ++++++ pkg/response/protocol/proto_version_test.go | 93 ++++++++ pkg/response/rcols/complex.go | 222 ++++++++++++++++++++ pkg/response/rcols/complex_test.go | 55 +++++ pkg/response/rcols/complex_unmarshal.go | 218 +++++++++++++++++++ pkg/response/rcols/init_column.go | 127 +++++++++++ pkg/response/rcols/interface.go | 31 +++ pkg/response/rcols/raw.go | 171 +++++++++++++++ pkg/response/rcols/raw_test.go | 29 +++ pkg/response/rcols/raw_unmarshal.go | 161 ++++++++++++++ pkg/response/rcols/utils.go | 60 ++++++ pkg/response/response.go | 111 ++++++++++ pkg/response/rrows/init_row.go | 91 ++++++++ pkg/response/rrows/row.go | 93 ++++++++ pkg/response/rrows/rows.go | 41 ++++ pkg/response/rrows/unmarshal.go | 29 +++ pkg/store/cqlstore.go | 5 +- pkg/store/helpers.go | 82 -------- pkg/store/store.go | 76 ++----- pkg/typedef/types.go | 4 + pkg/utils/utils.go | 16 ++ 29 files changed, 2257 insertions(+), 142 deletions(-) create mode 100644 pkg/response/comp/delete_equal.go create mode 100644 pkg/response/comp/delete_equal_test.go create mode 100644 pkg/response/comp/diff_info.go create mode 100644 pkg/response/data.go create mode 100644 pkg/response/data_test.go create mode 100644 pkg/response/diffs/helpers.go create mode 100644 pkg/response/diffs/info.go create mode 100644 pkg/response/interface.go create mode 100644 pkg/response/protocol/proto_version.go create mode 100644 pkg/response/protocol/proto_version_test.go create mode 100644 pkg/response/rcols/complex.go create mode 100644 pkg/response/rcols/complex_test.go create mode 100644 pkg/response/rcols/complex_unmarshal.go create mode 100644 pkg/response/rcols/init_column.go create mode 100644 pkg/response/rcols/interface.go create mode 100644 pkg/response/rcols/raw.go create mode 100644 pkg/response/rcols/raw_test.go create mode 100644 pkg/response/rcols/raw_unmarshal.go create mode 100644 pkg/response/rcols/utils.go create mode 100644 pkg/response/response.go create mode 100644 pkg/response/rrows/init_row.go create mode 100644 pkg/response/rrows/row.go create mode 100644 pkg/response/rrows/rows.go create mode 100644 pkg/response/rrows/unmarshal.go diff --git a/pkg/response/comp/delete_equal.go b/pkg/response/comp/delete_equal.go new file mode 100644 index 00000000..d20ca936 --- /dev/null +++ b/pkg/response/comp/delete_equal.go @@ -0,0 +1,61 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package comp + +import ( + "github.com/scylladb/gemini/pkg/response" + "github.com/scylladb/gemini/pkg/response/rrows" +) + +// easyDeleteEqualRows delete equal rows one by one. Its usual way, but not optimal. +// Fewer Rows must be in resp1. +func easyDeleteEqualRows(resp1, resp2 *response.Data) int { + idx := 0 + for _, row := range resp1.Rows { + if row.Equal(resp2.Rows[idx]) { + idx++ + continue + } + break + } + //delete rows + resp1.Rows = resp1.Rows[idx:] + resp2.Rows = resp2.Rows[idx:] + return idx +} + +// usualDeleteEqualRows delete equal rows. Its usual way, but not optimal. +// Fewer Rows must be in resp1. +func usualDeleteEqualRows(resp1, resp2 *response.Data) int { + equalRowsCount := easyDeleteEqualRows(resp1, resp2) + if len(resp1.Rows) == 0 || len(resp2.Rows) == 0 { + return equalRowsCount + } + idx1 := 0 + tmpOut := make(rrows.Rows, 0) + for _, row := range resp1.Rows { + if idx2 := resp2.Rows.FindEqual(*row); idx2 != -1 { + //delete rows + resp2.Rows[idx2] = resp2.Rows[len(resp2.Rows)-1] + resp2.Rows = resp2.Rows[:len(resp2.Rows)-1] + equalRowsCount++ + continue + } + tmpOut = append(tmpOut, row) + idx1++ + } + resp1.Rows = tmpOut + return equalRowsCount +} diff --git a/pkg/response/comp/delete_equal_test.go b/pkg/response/comp/delete_equal_test.go new file mode 100644 index 00000000..e9629c92 --- /dev/null +++ b/pkg/response/comp/delete_equal_test.go @@ -0,0 +1,107 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package comp + +import ( + "reflect" + "testing" + "time" + + "golang.org/x/exp/rand" + + "github.com/scylladb/gemini/pkg/response" + "github.com/scylladb/gemini/pkg/response/rcols" + "github.com/scylladb/gemini/pkg/response/rrows" + "github.com/scylladb/gemini/pkg/utils" +) + +var rnd = rand.New(rand.NewSource(uint64(time.Now().UnixMilli()))) + +func TestTryEasyDeleteEqualRows(t *testing.T) { + row1 := getRandomRawRow(20, 20) + row2 := getRandomRawRow(20, 20) + rows1 := getRandomRawRows(20, 20, 20) + rows2 := make(rrows.Rows, 20) + copy(rows2, rows1) + rows1[10] = &row1 + rows2[11] = &row2 + testRows1 := response.Data{ + Rows: rows1, + } + testRows2 := response.Data{ + Rows: rows2, + } + + expected1 := rows1[10:] + expected2 := rows2[10:] + + deleteCount := easyDeleteEqualRows(&testRows1, &testRows2) + + if !reflect.DeepEqual(testRows1.Rows, expected1) { + t.Fatalf("wrong easyDeleteEqualRows work. \nreceived:%+v \nexpected:%+v", testRows1.Rows, expected1) + } + if !reflect.DeepEqual(testRows2.Rows, expected2) { + t.Fatalf("wrong easyDeleteEqualRows work. \nreceived:%+v \nexpected:%+v", testRows1.Rows, expected2) + } + if deleteCount != 10 { + t.Fatalf("wrong easyDeleteEqualRows work. deletes count %d, but should %d ", deleteCount, 10) + } +} + +func TestTryUsualDeleteEqualRows(t *testing.T) { + row1 := getRandomRawRow(20, 20) + row2 := getRandomRawRow(20, 20) + rows1 := getRandomRawRows(20, 20, 20) + rows2 := make(rrows.Rows, 20) + copy(rows2, rows1) + rows1[10] = &row1 + rows2[10] = &row2 + + testRows1 := response.Data{ + Rows: rows1, + } + testRows2 := response.Data{ + Rows: rows2, + } + + usualDeleteEqualRows(&testRows1, &testRows2) + + if !reflect.DeepEqual(*testRows1.Rows[0], row1) || len(testRows1.Rows) != 1 { + t.Fatalf("wrong diff.deleteEqualRowsDiffLen work. \nreceived:%d \nexpected:%d", testRows1.LenRows(), 1) + } + if !reflect.DeepEqual(*testRows2.Rows[0], row2) || len(testRows2.Rows) != 1 { + t.Fatalf("wrong diff.deleteEqualRowsDiffLen work. \nreceived:%d \nexpected:%d", testRows2.LenColumns(), 1) + } +} + +func getRandomRawRows(rowsCount, columns, columnLen int) rrows.Rows { + out := make(rrows.Rows, rowsCount) + for idx := range out { + tmp := getRandomRawRow(columns, columnLen) + out[idx] = &tmp + } + return out +} + +func getRandomRawRow(columns, columnLen int) rrows.Row { + out := make(rrows.Row, columns) + for idx := range out { + col := utils.RandBytes(rnd, columnLen) + out[idx] = *(*rcols.ColumnLongRaw)(&col) + } + return out +} diff --git a/pkg/response/comp/diff_info.go b/pkg/response/comp/diff_info.go new file mode 100644 index 00000000..d10ee614 --- /dev/null +++ b/pkg/response/comp/diff_info.go @@ -0,0 +1,144 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package comp + +import ( + "fmt" + "strings" + + "github.com/scylladb/gemini/pkg/response" + "github.com/scylladb/gemini/pkg/response/diffs" +) + +// Info represent information about responses difference. +type Info []string + +func (d *Info) Len() int { + return len(*d) +} + +func (d *Info) Add(in ...string) { + *d = append(*d, in...) +} + +func (d *Info) String() string { + return strings.Join(*d, "\n") +} + +func (d *Info) addMissingRows(test, oracle *response.Data) { + if test.LenRows() != 0 { + d.Add("unequal test store rows:") + d.Add(test.RowsToStrings()...) + } + if oracle.LenRows() != 0 { + d.Add("unequal oracle store rows:") + d.Add(oracle.RowsToStrings()...) + } +} + +func (d *Info) AddUnequalRowsInfo(test, oracle *response.Data) { + d.Add(diffs.GetList(test, oracle).StringsRaw(test.Types, test.Names)...) +} + +// GetCompareInfo returns empty Info if responses are equal in other case returns information about responses difference. +func GetCompareInfo(test, oracle *response.Data, detailedDiff bool) Info { + diff := make(Info, 0) + lenTest := test.LenRows() + lenOracle := oracle.LenRows() + equalRowsCount := 0 + if lenTest == 0 && lenOracle == 0 { + // no rows in responses from test and oracle stores, nothing to compare. + return diff + } + if !detailedDiff { + if lenTest == 0 || lenOracle == 0 { + diff.Add(fmt.Sprintf("different rows count in responses: from test store-%d, from oracle store-%d", lenTest, lenOracle)) + return diff + } + if lenTest != lenOracle { + diff.Add(fmt.Sprintf("different rows count in responses: from test store-%d, from oracle store-%d", lenTest, lenOracle)) + return diff + } + equalRowsCount = deleteEqualRowsDiffLen(test, oracle) + if len(test.Rows) != 0 { + diff.Add(fmt.Sprintf("responses have %d equal rows and not equal rows %d", equalRowsCount, test.LenRows())) + } + return diff + } + + if lenTest == 0 || lenOracle == 0 { + diff.Add(fmt.Sprintf("different rows count in responses: from test store-%d, from oracle store-%d", lenTest, lenOracle)) + diff.addMissingRows(test, oracle) + return diff + } + // case with same len + if lenTest == lenOracle { + equalRowsCount = deleteEqualRowsSameLen(test, oracle) + if test.LenRows() == 0 { + return diff + } + diff.Add(fmt.Sprintf("responses have %d equal rows and not equal rows: test store %d; oracle store %d", equalRowsCount, test.LenRows(), oracle.LenRows())) + diff.AddUnequalRowsInfo(test, oracle) + return diff + } + // case with different len + equalRowsCount = deleteEqualRowsDiffLen(test, oracle) + diff.Add(fmt.Sprintf("responses have %d equal rows and not equal rows: test store %d; oracle store %d", equalRowsCount, test.LenRows(), oracle.LenRows())) + if test.LenRows() == 0 || oracle.LenRows() == 0 { + diff.addMissingRows(test, oracle) + return diff + } + diff.AddUnequalRowsInfo(test, oracle) + return diff +} + +// deleteEqualRowsDiffLen returns true if responses rows equal. +// Task of this function - chose optimal way for equal rows. +func deleteEqualRowsDiffLen(test, oracle *response.Data) int { + equalRowsCount := 0 + if test.LenRows() < oracle.LenRows() { + // case test rows fewer + equalRowsCount += easyDeleteEqualRows(test, oracle) + if test.LenRows() != 0 { + equalRowsCount += usualDeleteEqualRows(test, oracle) + } + return equalRowsCount + } + // case oracle rows fewer + equalRowsCount += easyDeleteEqualRows(oracle, test) + if oracle.LenRows() != 0 { + equalRowsCount += usualDeleteEqualRows(oracle, test) + } + return equalRowsCount +} + +// deleteEqualRowsDiffLen returns true if responses rows equal. +// Task of this function - chose optimal way for equal rows. +func deleteEqualRowsSameLen(test, oracle *response.Data) int { + if test.LenRows() == 1 { + if test.Rows[0].Equal(oracle.Rows[0]) { + //delete rows + test.Rows = nil + oracle.Rows = nil + return 1 + } + return 0 + } + equalRowsCount := easyDeleteEqualRows(test, oracle) + if test.LenRows() != 0 { + equalRowsCount += usualDeleteEqualRows(test, oracle) + } + return equalRowsCount +} diff --git a/pkg/response/data.go b/pkg/response/data.go new file mode 100644 index 00000000..d2a42416 --- /dev/null +++ b/pkg/response/data.go @@ -0,0 +1,54 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package response + +import ( + "github.com/gocql/gocql" + + "github.com/scylladb/gemini/pkg/response/rrows" +) + +type Data struct { + Types []gocql.TypeInfo + Names []string + Rows rrows.Rows +} + +func (d *Data) LenColumns() int { + return len(d.Names) +} + +func (d *Data) LenRows() int { + return len(d.Rows) +} + +func (d *Data) RowsToStrings() []string { + return d.Rows.Strings(d.Types, d.Names) +} + +func (d *Data) EqualColumns(d2 []string) bool { + if len(d.Names) != len(d2) { + return false + } + if len(d2) == 0 { + return true + } + for idx := range d.Names { + if d.Names[idx] != d2[idx] { + return false + } + } + return true +} diff --git a/pkg/response/data_test.go b/pkg/response/data_test.go new file mode 100644 index 00000000..6afbcd75 --- /dev/null +++ b/pkg/response/data_test.go @@ -0,0 +1,55 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package response + +import ( + "testing" + "time" + + "github.com/gocql/gocql" + "golang.org/x/exp/rand" + + "github.com/scylladb/gemini/pkg/typedef" + "github.com/scylladb/gemini/pkg/utils" +) + +var rnd = rand.New(rand.NewSource(uint64(time.Now().UnixMilli()))) + +func TestData_EqualHeader(t *testing.T) { + t.Parallel() + //header := randRowsHeader(30) +} + +func randRowsHeader(headerLen int) []*gocql.ColumnInfo { + header := make([]*gocql.ColumnInfo, headerLen) + for idx := range header { + header[idx] = &gocql.ColumnInfo{ + Keyspace: "test", + Table: "test", + Name: utils.RandString(rnd, 10), + TypeInfo: nil, + } + t := rnd.Intn(len(typedef.goCQLTypeMap)) + for _, val := range typedef.goCQLTypeMap { + if t == 0 { + header[idx].TypeInfo = val + } + t-- + } + } + return header +} diff --git a/pkg/response/diffs/helpers.go b/pkg/response/diffs/helpers.go new file mode 100644 index 00000000..f2f0e927 --- /dev/null +++ b/pkg/response/diffs/helpers.go @@ -0,0 +1,70 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package diffs + +import ( + "github.com/scylladb/gemini/pkg/response" + "github.com/scylladb/gemini/pkg/response/rrows" +) + +// getList returns list of unequal`s info +func getList(test, oracle *response.Data) List { + idx1 := 0 + list := make(List, 2) + + for range test.Rows { + if len(oracle.Rows) == 0 { + break + } + fewerUnequals, idx2 := getUnequalAndDelete(oracle.Rows, *test.Rows[idx1]) + if idx2 == -1 { + continue + } + list.put(info{ + colIDs: fewerUnequals, + test: *test.Rows[idx1], + oracle: *oracle.Rows[idx2], + sameRows: 1, + }) + + // delete rows + test.Rows[idx1] = test.Rows[len(test.Rows)-1] + test.Rows = test.Rows[:len(test.Rows)-1] + + oracle.Rows[idx2] = oracle.Rows[len(oracle.Rows)-1] + oracle.Rows = oracle.Rows[:len(oracle.Rows)-1] + + idx1++ + } + + return list +} + +func getUnequalAndDelete(rows rrows.Rows, testRow rrows.Row) (colIDs, int) { + fewerUnequalRowID := -1 + var fewerUnequalsTmp colIDs + fewerUnequals := make(colIDs, len(testRow)) + for idx := range rows { + fewerUnequalsTmp = testRow.UnequalColumnIDs(rows[idx], byte(len(fewerUnequals))) + if fewerUnequalsTmp == nil { + continue + } + if len(fewerUnequals) > len(fewerUnequalsTmp) { + fewerUnequals = fewerUnequalsTmp + fewerUnequalRowID = idx + } + } + return fewerUnequals, fewerUnequalRowID +} diff --git a/pkg/response/diffs/info.go b/pkg/response/diffs/info.go new file mode 100644 index 00000000..8287a57e --- /dev/null +++ b/pkg/response/diffs/info.go @@ -0,0 +1,91 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package diffs + +import ( + "fmt" + "github.com/scylladb/gemini/pkg/response" + + "github.com/gocql/gocql" + + "github.com/scylladb/gemini/pkg/response/rrows" +) + +type List map[string]*info + +type colIDs []byte + +func (c colIDs) string() string { + return string(c) +} + +func GetList(test, oracle *response.Data) List { + return getList(test, oracle) +} + +func (l List) put(results info) bool { + unequalName := results.colIDs.string() + if unequal, have := l[unequalName]; have { + unequal.sameRows++ + return false + } + l[unequalName] = &results + return true +} + +func (l List) StringsRaw(types []gocql.TypeInfo, names []string) []string { + if len(l) == 0 { + return nil + } + out := make([]string, len(l)*5+1) + out[0] = fmt.Sprintf("responses have %d different unequals", len(l)) + num := 1 + idx := 1 + for _, unequal := range l { + out[idx] = fmt.Sprintf("unequal type#%d have responses %d - same columns:%s", + num, unequal.sameRows, unequal.test.StringColumns(types, names, unequal.equalIDs())) + out[idx+1] = fmt.Sprintf("unequal type#%d test store unequal columns GO:%s", num, unequal.test.StringColumns(types, names, unequal.unequalIDs())) + out[idx+2] = fmt.Sprintf("unequal type#%d oracle store unequal columns GO:%s", num, unequal.oracle.StringColumns(types, names, unequal.unequalIDs())) + out[idx+3] = fmt.Sprintf("unequal type#%d test store unequal columns RAW:%s", num, unequal.test.StringColumnsRaw(names, unequal.unequalIDs())) + out[idx+4] = fmt.Sprintf("unequal type#%d oracle store unequal columns RAW:%s", num, unequal.oracle.StringColumnsRaw(names, unequal.unequalIDs())) + idx += 5 + num++ + } + return out +} + +type info struct { + colIDs colIDs + test rrows.Row + oracle rrows.Row + sameRows uint +} + +func (i *info) equalIDs() colIDs { + out := make(colIDs, 0) + for id := range i.test { + for _, unequalID := range i.colIDs { + if byte(id) == unequalID { + continue + } + } + out = append(out, byte(id)) + } + return out +} + +func (i *info) unequalIDs() colIDs { + return i.colIDs +} diff --git a/pkg/response/interface.go b/pkg/response/interface.go new file mode 100644 index 00000000..ca27a28f --- /dev/null +++ b/pkg/response/interface.go @@ -0,0 +1,32 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package response + +import ( + "github.com/gocql/gocql" +) + +type RowInterface interface { + Equal(row2 *RowInterface) bool + EqualOptimized(row2 *RowInterface, seq []byte) byte + UnequalColumnIDs(row2 *RowInterface, maxIDs byte) []byte + String(columns []gocql.ColumnInfo) string + StringColumns(columns []gocql.ColumnInfo, colIds []byte) string +} + +type RowsInterface interface { + EqualAndDelete(row RowInterface) bool + Strings(columns []gocql.ColumnInfo) []string +} diff --git a/pkg/response/protocol/proto_version.go b/pkg/response/protocol/proto_version.go new file mode 100644 index 00000000..9281211f --- /dev/null +++ b/pkg/response/protocol/proto_version.go @@ -0,0 +1,70 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protocol + +// protoDiffInfo represent information about ProtoVersions difference in responses. +// Main task - catch cases then stores or nodes of stores have different ProtoVersions. +type protoDiffInfo struct { + // count of responses checked on difference in ProtoVersions + count uint32 + init bool + version2orLess bool + detected bool + pass bool +} + +// responsesLimit represent max count of responses to check on difference in ProtoVersions. +const responsesLimit = uint32(200) + +var Diff = protoDiffInfo{ + count: 0, + init: false, + version2orLess: false, + pass: false, +} + +func (p *protoDiffInfo) reInit() { + p.count = 0 + p.init = false + + p.version2orLess = false + p.detected = false + p.pass = false +} + +func (p *protoDiffInfo) Detected() bool { + return p.detected +} + +func (p *protoDiffInfo) CheckDone() bool { + return p.pass +} + +func (p *protoDiffInfo) CheckVersions(version2orLess bool) { + if !p.init { + p.version2orLess = version2orLess + p.init = true + return + } + if version2orLess == p.version2orLess { + if Diff.count++; Diff.count >= responsesLimit { + p.pass = true + } + } else { + p.detected = true + p.pass = true + } + +} diff --git a/pkg/response/protocol/proto_version_test.go b/pkg/response/protocol/proto_version_test.go new file mode 100644 index 00000000..df60ec09 --- /dev/null +++ b/pkg/response/protocol/proto_version_test.go @@ -0,0 +1,93 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package protocol + +import ( + "sync" + "testing" + "time" + + "github.com/gocql/gocql" + "golang.org/x/exp/rand" + + "github.com/scylladb/gemini/pkg/typedef" +) + +var typeMap = typedef.GetGoCQLTypeMap() + +var rnd = rand.New(rand.NewSource(uint64(time.Now().UnixMilli()))) + +func TestProtoVersion(t *testing.T) { + t.Parallel() + // test without version diff + runParallel(100, 20, false) + if Diff.Detected() { + t.Fatalf("wrong Diff work, detected shoul be false, but get true") + } + if Diff.version2orLess || Diff.detected || !Diff.init || !Diff.pass { + t.Fatalf("wrong Diff work, version shoul be 4, but get 0") + } + // test version diff detect + Diff.reInit() + runParallel(100, 20, true) + if !Diff.Detected() { + t.Fatalf("wrong Diff work, detected shoul be true, but get false") + } + if !Diff.version2orLess || !Diff.init || !Diff.pass || !Diff.detected { + t.Fatalf("wrong Diff work, version shoul be 4, but get 0") + } +} + +func runParallel(parallel, headerLen int, pastDiff bool) { + wg := sync.WaitGroup{} + if pastDiff { + Diff.CheckVersions(true) + } + for i := 0; i < parallel; i++ { + wg.Add(1) + go func() { + l := 0 + defer wg.Done() + for { + Diff.CheckVersions(false) + l++ + if Diff.CheckDone() { + return + } + } + }() + } + wg.Wait() +} + +func randRowsHeader(headerLen int, wrongVersion bool) []gocql.TypeInfo { + header := make([]gocql.TypeInfo, headerLen) + for idx := range header { + t := rnd.Intn(len(typeMap)) + for _, val := range typeMap { + if t == 0 { + header[idx] = val + } + t-- + } + } + if wrongVersion { + idx := rnd.Intn(headerLen) + header[idx] = gocql.NewNativeType(typedef.GoCQLProtoVersion2, gocql.TypeText, "") + } + return header +} diff --git a/pkg/response/rcols/complex.go b/pkg/response/rcols/complex.go new file mode 100644 index 00000000..fbad0664 --- /dev/null +++ b/pkg/response/rcols/complex.go @@ -0,0 +1,222 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rcols + +import ( + "github.com/gocql/gocql" + "strconv" +) + +type Tuple []Column + +func (t Tuple) ToString(colInfo gocql.TypeInfo) string { + out := "tuple<" + if len(t) == 0 { + return out + ">" + } + tuple := colInfo.(gocql.TupleTypeInfo) + + for i, elem := range tuple.Elems { + out += strconv.Itoa(i) + ":" + t[i].ToString(elem) + ";" + } + out = out[:len(out)-1] + return out + ">" +} + +func (t Tuple) ToStringRaw() string { + out := "tuple<" + if len(t) == 0 { + return out + ">" + } + for i := range t { + out += strconv.Itoa(i) + ":" + t[i].ToStringRaw() + ";" + } + out = out[:len(out)-1] + return out + ">" +} + +func (t Tuple) Equal(colT interface{}) bool { + t2 := colT.(Tuple) + if len(t) != len(t2) { + return false + } + if len(t) == 0 { + return true + } + for idx, col := range t { + if !col.Equal(t2[idx]) { + return false + } + } + return true +} + +type Map []MapPair + +type MapPair struct { + Key Column + Value Column +} + +func (p MapPair) Equal(p2 MapPair) bool { + return p.Key.Equal(p2.Key) && p.Value.Equal(p2.Value) +} + +func (m Map) ToString(colInfo gocql.TypeInfo) string { + out := "map(" + if len(m) == 0 { + return out + ">" + } + mapInfo := colInfo.(gocql.CollectionType) + + for _, pair := range m { + out += pair.Key.ToString(mapInfo.Key) + ":" + pair.Value.ToString(mapInfo.Elem) + ";" + } + out = out[:len(out)-1] + return out + ")" +} + +func (m Map) ToStringRaw() string { + out := "map<" + if len(m) == 0 { + return out + ">" + } + for _, pair := range m { + out += pair.Key.ToStringRaw() + ":" + pair.Value.ToStringRaw() + ";" + } + out = out[:len(out)-1] + return out + ")" +} + +func (m Map) Equal(colT interface{}) bool { + m2 := colT.(Map) + if len(m) != len(m2) { + return false + } + if len(m) == 0 { + return true + } + for idx, pair := range m { + if !pair.Equal(m2[idx]) { + return false + } + } + return true +} + +type List []Column + +func (l List) ToString(colInfo gocql.TypeInfo) string { + out := "set<" + if colInfo.Type() == gocql.TypeList { + out = "list<" + } + if len(l) == 0 { + return out + ">" + } + listInfo := colInfo.(gocql.CollectionType) + + for idx := range l { + out += strconv.Itoa(idx) + ":" + l[idx].ToString(listInfo.Elem) + ";" + } + out = out[:len(out)-1] + return out + ">" +} + +func (l List) ToStringRaw() string { + out := "set||list<" + if len(l) == 0 { + return out + ">" + } + for idx := range l { + out += strconv.Itoa(idx) + ":" + l[idx].ToStringRaw() + ";" + } + out = out[:len(out)-1] + return out + ">" +} + +func (l List) Equal(colT interface{}) bool { + l2 := colT.(List) + if len(l) != len(l2) { + return false + } + if len(l) == 0 { + return true + } + for idx, col := range l { + if !col.Equal(l2[idx]) { + return false + } + } + return true +} + +type UDT struct { + Names []string + Values []Column +} + +func (u UDT) PairToString(idx int, colType gocql.TypeInfo) string { + return u.Names[idx] + ":" + u.Values[idx].ToString(colType) + ";" +} + +func (u UDT) PairToStringRaw(idx int) string { + return u.Names[idx] + ":" + u.Values[idx].ToStringRaw() + ";" +} + +func (u UDT) ToString(colInfo gocql.TypeInfo) string { + out := "udt<" + if len(u.Names) == 0 { + return out + ">" + } + udt := colInfo.(gocql.UDTTypeInfo) + + for idx := range u.Values { + out += u.PairToString(idx, udt.Elements[idx].Type) + } + out = out[:len(out)-1] + return out + ">" +} + +func (u UDT) ToStringRaw() string { + out := "udt<" + if len(u.Names) == 0 { + return out + ">" + } + for idx := range u.Values { + out += u.PairToStringRaw(idx) + } + out = out[:len(out)-1] + return out + ">" +} + +func (u UDT) Equal(colT interface{}) bool { + u2 := colT.(UDT) + if len(u.Names) != len(u2.Names) { + return false + } + if len(u.Names) == 0 { + return true + } + for idx, value := range u.Values { + if u.Names[idx] != u2.Names[idx] { + return false + } + if !value.Equal(u2.Values[idx]) { + return false + } + } + return true +} diff --git a/pkg/response/rcols/complex_test.go b/pkg/response/rcols/complex_test.go new file mode 100644 index 00000000..373b2ea1 --- /dev/null +++ b/pkg/response/rcols/complex_test.go @@ -0,0 +1,55 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package rcols + +import ( + "golang.org/x/exp/rand" + "reflect" + "testing" + "time" + + "github.com/scylladb/gemini/pkg/utils" +) + +var rnd = rand.New(rand.NewSource(uint64(time.Now().UnixMilli()))) + +func TestBytes_UnmarshalCQL(t *testing.T) { + t.Parallel() + errorMsg := "wrong UnmarshalCQL work:" + testBytes := make(ColumnLongRawUM, 0) + + testsCount := 1000 + for i := 0; i < testsCount; i++ { + expected := utils.RandBytes(rnd, rnd.Intn(1000)) + if i == 0 { + expected = ColumnLongRaw{} + } + _ = testBytes.UnmarshalCQL(nil, expected) + if message := helpEqualBytes(t, errorMsg, testBytes, expected); message != "" { + t.Fatal(message) + } + testBytes = make(ColumnLongRawUM, 0) + } +} + +func helpEqualBytes(t *testing.T, errorMsg string, received []byte, expected []byte) string { + t.Helper() + if !reflect.DeepEqual(expected, received) { + t.Fatalf("%s\nreceived:%+v \nexpected:%+v", errorMsg, received, expected) + } + return "" +} diff --git a/pkg/response/rcols/complex_unmarshal.go b/pkg/response/rcols/complex_unmarshal.go new file mode 100644 index 00000000..ff0f9d83 --- /dev/null +++ b/pkg/response/rcols/complex_unmarshal.go @@ -0,0 +1,218 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rcols + +import ( + "github.com/gocql/gocql" +) + +type TupleUM []ColumnUnmarshal + +func (t *TupleUM) UnmarshalCQL(colInfo gocql.TypeInfo, data []byte) error { + if len(data) == 0 { + t.ToNil() + return nil + } + tuple := colInfo.(gocql.TupleTypeInfo) + for i, elem := range *t { + var p []byte + if len(data) >= 4 { + p, data = readBytes(data) + } + err := elem.UnmarshalCQL(tuple.Elems[i], p) + if err != nil { + return err + } + } + return nil +} + +func (t *TupleUM) ToColumn() Column { + out := make(Tuple, len(*t)) + for idx, elem := range *t { + out[idx] = elem.ToColumn() + } + return out +} + +func (t *TupleUM) ToNil() { + for _, elem := range *t { + elem.ToNil() + } +} + +type MapUM struct { + Map Map + Key ColumnUnmarshal + Value ColumnUnmarshal +} + +func (m *MapUM) KeyAndValueToMap(idx int) { + m.Map[idx] = MapPair{ + Key: m.Key.ToColumn(), + Value: m.Value.ToColumn(), + } + m.Value.ToNil() + m.Key.ToNil() +} + +func (m *MapUM) UnmarshalCQL(colInfo gocql.TypeInfo, data []byte) error { + mapInfo := colInfo.(gocql.CollectionType) + if len(data) == 0 { + m.ToNil() + return nil + } + var err error + var trim, mapLen, keyLen, valLen int + version := colInfo.Version() + mapLen, trim, err = readCollectionSize(version, data) + if err != nil { + return err + } + m.Map = make(Map, mapLen) + data = data[trim:] + for idx := 0; idx < mapLen; idx++ { + keyLen, trim, err = readCollectionSize(version, data) + if err != nil { + return err + } + data = data[trim:] + if len(data) < keyLen { + return ErrorUnmarshalEOF + } + if err = m.Key.UnmarshalCQL(mapInfo.Key, data[:keyLen]); err != nil { + return err + } + data = data[keyLen:] + + valLen, trim, err = readCollectionSize(version, data) + if err != nil { + return err + } + data = data[trim:] + if len(data) < valLen { + return ErrorUnmarshalEOF + } + if err = m.Value.UnmarshalCQL(mapInfo.Elem, data[:valLen]); err != nil { + return err + } + data = data[valLen:] + m.KeyAndValueToMap(idx) + } + return nil +} + +func (m *MapUM) ToColumn() Column { + tmp := m.Map + m.ToNil() + return tmp +} + +func (m *MapUM) ToNil() { + m.Value.ToNil() + m.Key.ToNil() + m.Map = Map{} +} + +type ListUM struct { + List List + Elem ColumnUnmarshal +} + +func (l *ListUM) UnmarshalCQL(colInfo gocql.TypeInfo, data []byte) error { + listInfo := colInfo.(gocql.CollectionType) + if len(data) == 0 { + l.ToNil() + return nil + } + var err error + var listLen, elemLen, trim int + version := colInfo.Version() + listLen, trim, err = readCollectionSize(version, data) + if err != nil { + return err + } + data = data[trim:] + l.List = make(List, listLen) + for i := 0; i < listLen; i++ { + elemLen, trim, err = readCollectionSize(version, data) + if err != nil { + return err + } + data = data[trim:] + if len(data) < elemLen { + return ErrorUnmarshalEOF + } + if err = l.Elem.UnmarshalCQL(listInfo.Elem, data[:elemLen]); err != nil { + return err + } + l.List[i] = l.Elem.ToColumn() + data = data[elemLen:] + } + return nil +} + +func (l *ListUM) ToColumn() Column { + tmp := l.List + l.ToNil() + return tmp +} + +func (l *ListUM) ToNil() { + l.Elem.ToNil() + l.List = List{} +} + +type UDTum struct { + Names []string + Values []ColumnUnmarshal +} + +func (u *UDTum) UnmarshalCQL(colInfo gocql.TypeInfo, data []byte) error { + if len(data) == 0 { + u.ToNil() + return nil + } + udt := colInfo.(gocql.UDTTypeInfo) + for idx := range u.Values { + var p []byte + if len(data) >= 4 { + p, data = readBytes(data) + } + err := u.Values[idx].UnmarshalCQL(udt.Elements[idx].Type, p) + if err != nil { + return err + } + } + return nil +} + +func (u *UDTum) ToColumn() Column { + out := UDT{ + Names: make([]string, len(u.Values)), + Values: make([]Column, len(u.Values)), + } + copy(out.Names, u.Names) + for idx := range u.Values { + out.Values[idx] = u.Values[idx].ToColumn() + } + return out +} + +func (u *UDTum) ToNil() { + for idx := range u.Values { + u.Values[idx].ToNil() + } +} diff --git a/pkg/response/rcols/init_column.go b/pkg/response/rcols/init_column.go new file mode 100644 index 00000000..bd9ad1c8 --- /dev/null +++ b/pkg/response/rcols/init_column.go @@ -0,0 +1,127 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rcols + +import "github.com/gocql/gocql" + +func InitColumn(typeInfo gocql.TypeInfo) (ColumnUnmarshal, interface{}) { + switch typeInfo.Type() { + // Complex types that`s dependence from protoVersion + case gocql.TypeList, gocql.TypeSet: + return InitListOrSet(typeInfo) + case gocql.TypeMap: + return InitMap(typeInfo) + // Complex types that`s no dependence from protoVersion + case gocql.TypeTuple: + return InitTuple(typeInfo) + case gocql.TypeUDT: + return InitUDT(typeInfo) + // Text types with variant len + case gocql.TypeVarchar, gocql.TypeAscii, gocql.TypeText, gocql.TypeBlob: + tmp := new(ColumnLongRawUM) + return tmp, tmp + // Types with variant len + case gocql.TypeDuration, gocql.TypeInet, gocql.TypeDecimal, gocql.TypeVarint: + tmp := new(ColumnLongRawUM) + return tmp, tmp + // Types with len =16 + case gocql.TypeUUID, gocql.TypeTimeUUID: + tmp := new(Column16RawUM) + return tmp, tmp + // Types with len =8 + case gocql.TypeBigInt, gocql.TypeCounter, gocql.TypeTime, gocql.TypeTimestamp, gocql.TypeDouble: + tmp := new(Column8RawUM) + return tmp, tmp + // Types with len =4 + case gocql.TypeFloat, gocql.TypeInt, gocql.TypeDate: + tmp := new(Column4RawUM) + return tmp, tmp + // Types with len =2 + case gocql.TypeSmallInt: + tmp := new(Column2RawUM) + return tmp, tmp + // Types with len =1 + case gocql.TypeTinyInt: + tmp := new(Column1RawUM) + return tmp, tmp + case gocql.TypeBoolean: + tmp := new(ColumnBoolUM) + return tmp, tmp + default: + panic("unknown column type") + } +} + +func InitTuple(colInfo gocql.TypeInfo) (ColumnUnmarshal, interface{}) { + tuple := colInfo.(gocql.TupleTypeInfo) + haveDep := false + // check for cases that`s dependence from protoVersion + for idx := range tuple.Elems { + elemType := tuple.Elems[idx].Type() + if elemType == gocql.TypeList || elemType == gocql.TypeSet || elemType == gocql.TypeMap { + haveDep = true + } + } + if !haveDep { + out := new(ColumnLongRawUM) + return out, out + } + var out *TupleUM + *out = make(TupleUM, len(tuple.Elems)) + for i, elem := range tuple.Elems { + (*out)[i], _ = InitColumn(elem) + } + return out, out +} + +func InitMap(colInfo gocql.TypeInfo) (ColumnUnmarshal, interface{}) { + mapInfo := colInfo.(gocql.CollectionType) + out := new(MapUM) + out.Key, _ = InitColumn(mapInfo.Key) + out.Value, _ = InitColumn(mapInfo.Elem) + return out, out +} + +func InitListOrSet(colInfo gocql.TypeInfo) (ColumnUnmarshal, interface{}) { + list := colInfo.(gocql.CollectionType) + out := new(ListUM) + out.List = make(List, 0) + out.Elem, _ = InitColumn(list.Elem) + return out, out +} + +func InitUDT(colInfo gocql.TypeInfo) (ColumnUnmarshal, interface{}) { + udt := colInfo.(gocql.UDTTypeInfo) + haveDep := false + // check for cases that`s dependence from protoVersion + for idx := range udt.Elements { + elemType := udt.Elements[idx].Type.Type() + if elemType == gocql.TypeList || elemType == gocql.TypeSet || elemType == gocql.TypeMap { + haveDep = true + } + } + if !haveDep { + out := new(ColumnLongRawUM) + return out, out + } + out := new(UDTum) + out.Names = make([]string, len(udt.Elements)) + out.Values = make([]ColumnUnmarshal, len(udt.Elements)) + for i, elem := range udt.Elements { + out.Names[i] = elem.Name + out.Values[i], _ = InitColumn(elem.Type) + } + return out, out +} diff --git a/pkg/response/rcols/interface.go b/pkg/response/rcols/interface.go new file mode 100644 index 00000000..0ef08cb3 --- /dev/null +++ b/pkg/response/rcols/interface.go @@ -0,0 +1,31 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rcols + +import ( + "github.com/gocql/gocql" +) + +type Column interface { + Equal(interface{}) bool + ToString(gocql.TypeInfo) string + ToStringRaw() string +} + +type ColumnUnmarshal interface { + UnmarshalCQL(_ gocql.TypeInfo, data []byte) error + ToColumn() Column + ToNil() +} diff --git a/pkg/response/rcols/raw.go b/pkg/response/rcols/raw.go new file mode 100644 index 00000000..85c72760 --- /dev/null +++ b/pkg/response/rcols/raw.go @@ -0,0 +1,171 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rcols + +import ( + "fmt" + "strconv" + "unsafe" + + "github.com/gocql/gocql" +) + +// ColumnLongRaw for case with difference len. +type ColumnLongRaw []byte + +func (col ColumnLongRaw) ToString(colInfo gocql.TypeInfo) string { + if len(col) == 0 { + return "" + } + tmpVal := colInfo.New() + if err := gocql.Unmarshal(colInfo, col, tmpVal); err != nil { + panic(err) + } + switch colInfo.Type() { + // Complex types that`s dependence from protoVersion + case gocql.TypeList: + return fmt.Sprintf("list<%v>", dereference(tmpVal)) + case gocql.TypeSet: + return fmt.Sprintf("set<%v>", dereference(tmpVal)) + case gocql.TypeMap: + return fmt.Sprintf("map<%v>", dereference(tmpVal)) + case gocql.TypeTuple: + return fmt.Sprintf("tuple<%v>", dereference(tmpVal)) + case gocql.TypeUDT: + return fmt.Sprintf("udt<%v>", dereference(tmpVal)) + default: + return fmt.Sprintf("%v", dereference(tmpVal)) + } +} + +func (col ColumnLongRaw) ToStringRaw() string { + return fmt.Sprint(col) +} + +func (col ColumnLongRaw) Equal(colT interface{}) bool { + return string(col) == string(colT.(ColumnLongRaw)) +} + +// Column16Raw for case with len=16. +type Column16Raw [16]byte + +func (col Column16Raw) ToString(colInfo gocql.TypeInfo) string { + tmpVal := colInfo.New() + if err := gocql.Unmarshal(colInfo, unsafe.Slice(&col[0], 16), tmpVal); err != nil { + panic(err) + } + return fmt.Sprintf("%+v", dereference(tmpVal)) +} + +func (col Column16Raw) ToStringRaw() string { + return fmt.Sprint(unsafe.Slice(&col[0], 16)) +} + +func (col Column16Raw) Equal(colT interface{}) bool { + return col == colT.(Column16Raw) +} + +// Column8Raw for case with len=8. +type Column8Raw [8]byte + +func (col Column8Raw) ToString(colInfo gocql.TypeInfo) string { + tmpVal := colInfo.New() + if err := gocql.Unmarshal(colInfo, unsafe.Slice(&col[0], 8), tmpVal); err != nil { + panic(err) + } + return fmt.Sprintf("%+v", dereference(tmpVal)) +} + +func (col Column8Raw) ToStringRaw() string { + return fmt.Sprint(unsafe.Slice(&col[0], 8)) +} + +func (col Column8Raw) Equal(colT interface{}) bool { + return col == colT.(Column8Raw) +} + +// Column4Raw for case with len=4. +type Column4Raw [4]byte + +func (col Column4Raw) ToString(colInfo gocql.TypeInfo) string { + tmpVal := colInfo.New() + if err := gocql.Unmarshal(colInfo, unsafe.Slice(&col[0], 4), tmpVal); err != nil { + panic(err) + } + return fmt.Sprintf("%+v", dereference(tmpVal)) +} + +func (col Column4Raw) ToStringRaw() string { + return fmt.Sprint(unsafe.Slice(&col[0], 4)) +} + +func (col Column4Raw) Equal(colT interface{}) bool { + return col == colT.(Column4Raw) +} + +// Column2Raw for cases with len=2. +type Column2Raw [2]byte + +func (col Column2Raw) ToString(colInfo gocql.TypeInfo) string { + tmpVal := colInfo.New() + if err := gocql.Unmarshal(colInfo, unsafe.Slice(&col[0], 2), tmpVal); err != nil { + panic(err) + } + return fmt.Sprintf("%+v", dereference(tmpVal)) +} + +func (col Column2Raw) ToStringRaw() string { + return fmt.Sprint(unsafe.Slice(&col[0], 2)) +} + +func (col Column2Raw) Equal(colT interface{}) bool { + return col == colT.(Column2Raw) +} + +// Column1Raw for cases with len=1. +type Column1Raw byte + +func (col Column1Raw) ToString(_ gocql.TypeInfo) string { + return strconv.Itoa(int(col)) +} + +func (col Column1Raw) ToStringRaw() string { + return fmt.Sprint(col) +} + +func (col Column1Raw) Equal(colT interface{}) bool { + return col == colT.(Column1Raw) +} + +// ColumnBool for bool cases. +type ColumnBool bool + +func (col ColumnBool) ToString(_ gocql.TypeInfo) string { + if col { + return "true" + } + return "false" +} + +func (col ColumnBool) ToStringRaw() string { + if col { + return "1" + } + return "0" +} + +func (col ColumnBool) Equal(col2 interface{}) bool { + return col == col2.(ColumnBool) +} diff --git a/pkg/response/rcols/raw_test.go b/pkg/response/rcols/raw_test.go new file mode 100644 index 00000000..0939fad1 --- /dev/null +++ b/pkg/response/rcols/raw_test.go @@ -0,0 +1,29 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:thelper + +package rcols + +import ( + "github.com/scylladb/gemini/pkg/utils" + "testing" +) + +func TestToStringRAW(t *testing.T) { + t.Parallel() + testBytes := make(ColumnLongRaw, 0) + testBytes = utils.RandBytes(rnd, rnd.Intn(10)) + print(testBytes.ToStringRaw()) +} diff --git a/pkg/response/rcols/raw_unmarshal.go b/pkg/response/rcols/raw_unmarshal.go new file mode 100644 index 00000000..5aa0e469 --- /dev/null +++ b/pkg/response/rcols/raw_unmarshal.go @@ -0,0 +1,161 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rcols + +import ( + "unsafe" + + "github.com/gocql/gocql" +) + +// ColumnLongRawUM for unmarshall ColumnLongRaw. +type ColumnLongRawUM ColumnLongRaw + +func (col *ColumnLongRawUM) UnmarshalCQL(_ gocql.TypeInfo, data []byte) error { + if len(data) == 0 { + col.ToNil() + return nil + } + *col = *(*ColumnLongRawUM)(&data) + return nil +} + +func (col *ColumnLongRawUM) ToColumn() Column { + return *(*ColumnLongRaw)(col) +} + +func (col *ColumnLongRawUM) ToNil() { + *col = ColumnLongRawUM{} +} + +// Column16RawUM for unmarshall Column16Raw. +type Column16RawUM Column16Raw + +func (col *Column16RawUM) UnmarshalCQL(_ gocql.TypeInfo, data []byte) error { + if len(data) == 0 { + col.ToNil() + return nil + } + *col = *(*Column16RawUM)(unsafe.Pointer(&data[0])) + return nil +} + +func (col *Column16RawUM) ToColumn() Column { + return *(*Column16Raw)(col) +} + +func (col *Column16RawUM) ToNil() { + *col = Column16RawUM{} +} + +// Column8RawUM for unmarshall Column8Raw. +type Column8RawUM [8]byte + +func (col *Column8RawUM) UnmarshalCQL(_ gocql.TypeInfo, data []byte) error { + if len(data) == 0 { + col.ToNil() + return nil + } + *col = *(*Column8RawUM)(unsafe.Pointer(&data[0])) + return nil +} + +func (col *Column8RawUM) ToColumn() Column { + return *(*Column8Raw)(col) +} + +func (col *Column8RawUM) ToNil() { + *col = Column8RawUM{} +} + +// Column4RawUM for unmarshall Column4Raw. +type Column4RawUM [4]byte + +func (col *Column4RawUM) UnmarshalCQL(_ gocql.TypeInfo, data []byte) error { + if len(data) == 0 { + col.ToNil() + return nil + } + *col = *(*Column4RawUM)(unsafe.Pointer(&data[0])) + return nil +} + +func (col *Column4RawUM) ToColumn() Column { + return *(*Column4Raw)(col) +} + +func (col *Column4RawUM) ToNil() { + *col = Column4RawUM{} +} + +// Column2RawUM for unmarshall Column2Raw. +type Column2RawUM [2]byte + +func (col *Column2RawUM) UnmarshalCQL(_ gocql.TypeInfo, data []byte) error { + if len(data) == 0 { + col.ToNil() + return nil + } + *col = *(*Column2RawUM)(unsafe.Pointer(&data[0])) + return nil +} + +func (col *Column2RawUM) ToColumn() Column { + return *(*Column2Raw)(col) +} + +func (col *Column2RawUM) ToNil() { + *col = Column2RawUM{} +} + +// Column1RawUM for unmarshall Column1Raw. +type Column1RawUM byte + +func (col *Column1RawUM) UnmarshalCQL(_ gocql.TypeInfo, data []byte) error { + if len(data) == 0 { + col.ToNil() + return nil + } + *col = *(*Column1RawUM)(&data[0]) + return nil +} + +func (col *Column1RawUM) ToColumn() Column { + return *(*Column1Raw)(col) +} + +func (col *Column1RawUM) ToNil() { + *col = Column1RawUM(0) +} + +// ColumnBoolUM for unmarshall ColumnBool. +type ColumnBoolUM bool + +func (col *ColumnBoolUM) UnmarshalCQL(_ gocql.TypeInfo, data []byte) error { + if len(data) == 0 { + col.ToNil() + return nil + } + *col = data[0] == 1 + return nil +} + +func (col *ColumnBoolUM) ToColumn() Column { + return *(*ColumnBool)(col) +} + +func (col *ColumnBoolUM) ToNil() { + *col = false +} diff --git a/pkg/response/rcols/utils.go b/pkg/response/rcols/utils.go new file mode 100644 index 00000000..00edd4e5 --- /dev/null +++ b/pkg/response/rcols/utils.go @@ -0,0 +1,60 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rcols + +import ( + "github.com/pkg/errors" + "reflect" +) + +func dereference(in interface{}) interface{} { + return reflect.Indirect(reflect.ValueOf(in)).Interface() +} + +func readBytes(p []byte) ([]byte, []byte) { + // TODO: really should use a framer + size := readInt(p) + p = p[4:] + if size < 0 { + return nil, p + } + return p[:size], p[size:] +} + +const protoVersion2 = 0x02 + +func readInt(p []byte) int32 { + return int32(p[0])<<24 | int32(p[1])<<16 | int32(p[2])<<8 | int32(p[3]) +} + +var ErrorUnmarshalEOF = errors.New("unexpected eof") + +func readCollectionSize(protoVersion byte, data []byte) (size, read int, err error) { + if protoVersion <= protoVersion2 { + if len(data) < 2 { + return 0, 0, ErrorUnmarshalEOF + } + size = int(data[0])<<8 | int(data[1]) + read = 2 + return + } + + if len(data) < 4 { + return 0, 0, ErrorUnmarshalEOF + } + size = int(data[0])<<24 | int(data[1])<<16 | int(data[2])<<8 | int(data[3]) + read = 4 + return +} diff --git a/pkg/response/response.go b/pkg/response/response.go new file mode 100644 index 00000000..1a8e3a4e --- /dev/null +++ b/pkg/response/response.go @@ -0,0 +1,111 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package response + +import ( + "fmt" + "github.com/gocql/gocql" + + "github.com/scylladb/gemini/pkg/response/protocol" + "github.com/scylladb/gemini/pkg/response/rrows" +) + +// GetResponse returns columns info,rows data (as go type or as raw data). +func GetResponse(iter *gocql.Iter, differentProtocols bool) Data { + if iter.NumRows() == 0 { + return Data{} + } + if differentProtocols { + rows, types, names := scanRowsRawDiffProtocol(iter) + return Data{ + Rows: rows, + Types: types, + Names: names, + } + } + rows, types, names := scanRowsRaw(iter) + return Data{ + Rows: rows, + Types: types, + Names: names, + } +} + +// scanRowsRaw returns rows data as Raw types. +func scanRowsRaw(iter *gocql.Iter) (rrows.Rows, []gocql.TypeInfo, []string) { + types, names := getColumnsInfo(iter) + if !protocol.Diff.CheckDone() { + protocol.Diff.CheckVersions(types[0].Version() <= 2) + } + count := 0 + baseRow, refRow := rrows.InitRawRow(types) + out := make(rrows.Rows, 2) + + for iter.Scan(refRow...) { + if count > len(out)-1 { + out = append(out, out...) + } + out[count] = baseRow.Dereference() + count++ + } + out = out[:count] + + return out, types, names +} + +// scanRowsRaw returns rows data as Raw types. +func scanRowsRawDiffProtocol(iter *gocql.Iter) (rrows.Rows, []gocql.TypeInfo, []string) { + types, names := getColumnsInfo(iter) + count := 0 + baseRow, refRow := rrows.InitDiffVersionRawRow(types) + out := make(rrows.Rows, 2) + + for iter.Scan(refRow...) { + if count > len(out)-1 { + out = append(out, out...) + } + out[count] = baseRow.Dereference() + count++ + } + out = out[:count] + + return out, types, names +} + +// getColumnsInfo returns columns info. Code from gocql. +func getColumnsInfo(iter *gocql.Iter) ([]gocql.TypeInfo, []string) { + types := make([]gocql.TypeInfo, len(iter.Columns())) + names := make([]string, len(iter.Columns())) + idx := 0 + for _, column := range iter.Columns() { + if col, ok := column.TypeInfo.(gocql.TupleTypeInfo); ok { + tmpTypes := make([]gocql.TypeInfo, len(col.Elems)-1) + tmpNames := make([]string, len(col.Elems)-1) + types = append(types, tmpTypes...) + names = append(names, tmpNames...) + for i := range col.Elems { + types[idx] = col.Elems[i] + names[idx] = fmt.Sprintf("%s.t[%d]", column.Name, i) + idx++ + } + } else { + types[idx] = column.TypeInfo + names[idx] = column.Name + idx++ + } + } + + return types, names +} diff --git a/pkg/response/rrows/init_row.go b/pkg/response/rrows/init_row.go new file mode 100644 index 00000000..ca0f26a7 --- /dev/null +++ b/pkg/response/rrows/init_row.go @@ -0,0 +1,91 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rrows + +import ( + "github.com/gocql/gocql" + + "github.com/scylladb/gemini/pkg/response/rcols" +) + +func InitRawRow(types []gocql.TypeInfo) (RowUnmarshal, []interface{}) { + baseRow := make(RowUnmarshal, len(types)) + refRow := make([]interface{}, len(types)) + for idx := range types { + switch types[idx].Type() { + // Complex types that`s dependence from protoVersion + case gocql.TypeList, gocql.TypeSet, gocql.TypeMap: + tmp := new(rcols.ColumnLongRawUM) + refRow[idx] = tmp + baseRow[idx] = tmp + // Complex types that`s no dependence from protoVersion + case gocql.TypeTuple, gocql.TypeUDT: + tmp := new(rcols.ColumnLongRawUM) + refRow[idx] = tmp + baseRow[idx] = tmp + // Text types with variant len + case gocql.TypeVarchar, gocql.TypeAscii, gocql.TypeText, gocql.TypeBlob: + tmp := new(rcols.ColumnLongRawUM) + refRow[idx] = tmp + baseRow[idx] = tmp + // Types with variant len + case gocql.TypeDuration, gocql.TypeInet, gocql.TypeDecimal, gocql.TypeVarint: + tmp := new(rcols.ColumnLongRawUM) + refRow[idx] = tmp + baseRow[idx] = tmp + // Types with len =16 + case gocql.TypeUUID, gocql.TypeTimeUUID: + tmp := new(rcols.Column16RawUM) + refRow[idx] = tmp + baseRow[idx] = tmp + // Types with len =8 + case gocql.TypeBigInt, gocql.TypeCounter, gocql.TypeTime, gocql.TypeTimestamp, gocql.TypeDouble: + tmp := new(rcols.Column8RawUM) + refRow[idx] = tmp + baseRow[idx] = tmp + // Types with len =4 + case gocql.TypeFloat, gocql.TypeInt, gocql.TypeDate: + tmp := new(rcols.Column4RawUM) + refRow[idx] = tmp + baseRow[idx] = tmp + // Types with len =2 + case gocql.TypeSmallInt: + tmp := new(rcols.Column2RawUM) + refRow[idx] = tmp + baseRow[idx] = tmp + // Types with len =1 + case gocql.TypeTinyInt: + tmp := new(rcols.Column1RawUM) + refRow[idx] = tmp + baseRow[idx] = tmp + case gocql.TypeBoolean: + tmp := new(rcols.ColumnBoolUM) + refRow[idx] = tmp + baseRow[idx] = tmp + default: + panic("unknown column type") + } + } + return baseRow, refRow +} + +func InitDiffVersionRawRow(types []gocql.TypeInfo) (RowUnmarshal, []interface{}) { + baseRow := make(RowUnmarshal, len(types)) + refRow := make([]interface{}, len(types)) + for idx := range types { + baseRow[idx], refRow[idx] = rcols.InitColumn(types[idx]) + } + return baseRow, refRow +} diff --git a/pkg/response/rrows/row.go b/pkg/response/rrows/row.go new file mode 100644 index 00000000..5e4a91c0 --- /dev/null +++ b/pkg/response/rrows/row.go @@ -0,0 +1,93 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rrows + +import ( + "fmt" + + "github.com/gocql/gocql" + + "github.com/scylladb/gemini/pkg/response/rcols" +) + +type Row []rcols.Column + +func (row *Row) Len() int { + return len(*row) +} + +func (row *Row) String(types []gocql.TypeInfo, names []string) string { + out := "" + if len(*row) == 0 { + return out + } + for idx := range *row { + out += fmt.Sprintf("%s:%+v", names[idx], (*row)[idx].ToString(types[idx])) + columnsSeparator + } + return out[:len(out)-1] +} + +func (row *Row) StringColumns(types []gocql.TypeInfo, names []string, colIds []byte) string { + out := "" + if len(colIds) == 0 { + return out + } + for _, idx := range colIds { + out += fmt.Sprintf("%s:%+v", names[idx], (*row)[idx].ToString(types[idx])) + columnsSeparator + } + return out[:len(out)-1] +} + +func (row *Row) StringColumnsRaw(names []string, colIds []byte) string { + out := "" + if len(colIds) == 0 { + return out + } + for _, idx := range colIds { + out += fmt.Sprintf("%s:%+v", names[idx], (*row)[idx].ToStringRaw()) + columnsSeparator + } + return out[:len(out)-1] +} + +func (row *Row) Equal(row2 *Row) bool { + if len(*row) != len(*row2) { + return false + } + if len(*row) == 0 { + return true + } + for idx := range *row { + if !(*row2)[idx].Equal((*row)[idx]) { + return false + } + } + return true +} + +// UnequalColumnIDs returns all unequal column id`s. +func (row *Row) UnequalColumnIDs(row2 *Row, maxIDs byte) []byte { + unequalIDs := make([]byte, 0, maxIDs) + count := byte(0) + for idx := range *row { + if !(*row2)[idx].Equal((*row)[idx]) { + unequalIDs = append(unequalIDs, byte(idx)) + count++ + } + if count == maxIDs { + return nil + } + } + return unequalIDs +} diff --git a/pkg/response/rrows/rows.go b/pkg/response/rrows/rows.go new file mode 100644 index 00000000..2dd329a8 --- /dev/null +++ b/pkg/response/rrows/rows.go @@ -0,0 +1,41 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rrows + +import ( + "fmt" + "github.com/gocql/gocql" +) + +const columnsSeparator = ";" + +type Rows []*Row + +func (rows Rows) Strings(types []gocql.TypeInfo, names []string) []string { + out := make([]string, len(rows)) + for idx := range rows { + out[idx] = fmt.Sprintf("row%d:%s", idx, rows[idx].String(types, names)) + } + return out +} + +func (rows Rows) FindEqual(row Row) int { + for idx := range rows { + if row.Equal(rows[idx]) { + return idx + } + } + return -1 +} diff --git a/pkg/response/rrows/unmarshal.go b/pkg/response/rrows/unmarshal.go new file mode 100644 index 00000000..ab9faa1e --- /dev/null +++ b/pkg/response/rrows/unmarshal.go @@ -0,0 +1,29 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rrows + +import ( + "github.com/scylladb/gemini/pkg/response/rcols" +) + +type RowUnmarshal []rcols.ColumnUnmarshal + +func (row *RowUnmarshal) Dereference() *Row { + out := make(Row, len(*row)) + for idx := range out { + out[idx] = (*row)[idx].ToColumn() + } + return &out +} diff --git a/pkg/store/cqlstore.go b/pkg/store/cqlstore.go index 82a99868..6644d645 100644 --- a/pkg/store/cqlstore.go +++ b/pkg/store/cqlstore.go @@ -27,6 +27,7 @@ import ( "github.com/scylladb/gocqlx/v2/qb" "go.uber.org/zap" + "github.com/scylladb/gemini/pkg/response" "github.com/scylladb/gemini/pkg/typedef" ) @@ -90,11 +91,11 @@ func (cs *cqlStore) doMutate(ctx context.Context, builder qb.Builder, ts time.Ti return nil } -func (cs *cqlStore) load(ctx context.Context, builder qb.Builder, values []interface{}) (result []map[string]interface{}, err error) { +func (cs *cqlStore) load(ctx context.Context, builder qb.Builder, differentProtocols bool, values []interface{}) (response.Data, error) { query, _ := builder.ToCql() iter := cs.session.Query(query, values...).WithContext(ctx).Iter() cs.ops.WithLabelValues(cs.system, opType(builder)).Inc() - return loadSet(iter), iter.Close() + return response.GetResponse(iter, differentProtocols), iter.Close() } func (cs cqlStore) close() error { diff --git a/pkg/store/helpers.go b/pkg/store/helpers.go index c95546d5..36c5e581 100644 --- a/pkg/store/helpers.go +++ b/pkg/store/helpers.go @@ -13,85 +13,3 @@ // limitations under the License. package store - -import ( - "fmt" - "math/big" - "strings" - "time" - - "github.com/gocql/gocql" - - "github.com/scylladb/gemini/pkg/typedef" -) - -func pks(t *typedef.Table, rows []map[string]interface{}) []string { - var keySet []string - for _, row := range rows { - keys := make([]string, 0, len(t.PartitionKeys)+len(t.ClusteringKeys)) - keys = extractRowValues(keys, t.PartitionKeys, row) - keys = extractRowValues(keys, t.ClusteringKeys, row) - keySet = append(keySet, strings.Join(keys, ", ")) - } - return keySet -} - -func extractRowValues(values []string, columns typedef.Columns, row map[string]interface{}) []string { - for _, pk := range columns { - values = append(values, fmt.Sprintf(pk.Name+"=%v", row[pk.Name])) - } - return values -} - -func lt(mi, mj map[string]interface{}) bool { - switch mis := mi["pk0"].(type) { - case []byte: - mjs, _ := mj["pk0"].([]byte) - return string(mis) < string(mjs) - case string: - mjs, _ := mj["pk0"].(string) - return mis < mjs - case int: - mjs, _ := mj["pk0"].(int) - return mis < mjs - case int8: - mjs, _ := mj["pk0"].(int8) - return mis < mjs - case int16: - mjs, _ := mj["pk0"].(int16) - return mis < mjs - case int32: - mjs, _ := mj["pk0"].(int32) - return mis < mjs - case int64: - mjs, _ := mj["pk0"].(int64) - return mis < mjs - case gocql.UUID: - mjs, _ := mj["pk0"].(gocql.UUID) - return mis.String() < mjs.String() - case time.Time: - mjs, _ := mj["pk0"].(time.Time) - return mis.UnixNano() < mjs.UnixNano() - case *big.Int: - mjs, _ := mj["pk0"].(*big.Int) - return mis.Cmp(mjs) < 0 - case nil: - return true - default: - msg := fmt.Sprintf("unhandled type %T\n", mis) - time.Sleep(time.Second) - panic(msg) - } -} - -func loadSet(iter *gocql.Iter) []map[string]interface{} { - var rows []map[string]interface{} - for { - row := make(map[string]interface{}) - if !iter.MapScan(row) { - break - } - rows = append(rows, row) - } - return rows -} diff --git a/pkg/store/store.go b/pkg/store/store.go index ed5fd98a..fd56e170 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -16,32 +16,28 @@ package store import ( "context" - "fmt" - "math/big" "os" - "reflect" - "sort" "sync" "time" - "go.uber.org/zap" - "github.com/gocql/gocql" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/scylladb/go-set/strset" "github.com/scylladb/gocqlx/v2/qb" "go.uber.org/multierr" - "gopkg.in/inf.v0" + "go.uber.org/zap" + "github.com/scylladb/gemini/pkg/response" + "github.com/scylladb/gemini/pkg/response/comp" + "github.com/scylladb/gemini/pkg/response/protocol" "github.com/scylladb/gemini/pkg/typedef" ) +var errorResponseDiffer = errors.New("response from test and oracle store have difference") + type loader interface { - load(context.Context, qb.Builder, []interface{}) ([]map[string]interface{}, error) + load(context.Context, qb.Builder, bool, []interface{}) (response.Data, error) } type storer interface { @@ -129,8 +125,8 @@ func (n *noOpStore) mutate(context.Context, qb.Builder, ...interface{}) error { return nil } -func (n *noOpStore) load(context.Context, qb.Builder, []interface{}) ([]map[string]interface{}, error) { - return nil, nil +func (n *noOpStore) load(context.Context, qb.Builder, bool, []interface{}) (response.Data, error) { + return response.Data{}, nil } func (n *noOpStore) Close() error { @@ -191,16 +187,17 @@ func mutate(ctx context.Context, s storeLoader, builder qb.Builder, values ...in return nil } -func (ds delegatingStore) Check(ctx context.Context, table *typedef.Table, builder qb.Builder, detailedDiff bool, values ...interface{}) error { - var testRows, oracleRows []map[string]interface{} +func (ds delegatingStore) Check(ctx context.Context, _ *typedef.Table, builder qb.Builder, detailedDiff bool, values ...interface{}) error { + var testRows, oracleRows response.Data var testErr, oracleErr error var wg sync.WaitGroup wg.Add(1) + diffProtocol := protocol.Diff.Detected() go func() { - testRows, testErr = ds.testStore.load(ctx, builder, values) + testRows, testErr = ds.testStore.load(ctx, builder, diffProtocol, values) wg.Done() }() - oracleRows, oracleErr = ds.oracleStore.load(ctx, builder, values) + oracleRows, oracleErr = ds.oracleStore.load(ctx, builder, diffProtocol, values) if oracleErr != nil { return errors.Wrapf(oracleErr, "unable to load check data from the oracle store") } @@ -211,49 +208,12 @@ func (ds delegatingStore) Check(ctx context.Context, table *typedef.Table, build if !ds.validations { return nil } - if len(testRows) == 0 && len(oracleRows) == 0 { - return nil - } - if len(testRows) != len(oracleRows) { - if !detailedDiff { - return fmt.Errorf("rows count differ (test store rows %d, oracle store rows %d, detailed information will be at last attempt)", len(testRows), len(oracleRows)) - } - testSet := strset.New(pks(table, testRows)...) - oracleSet := strset.New(pks(table, oracleRows)...) - missingInTest := strset.Difference(oracleSet, testSet).List() - missingInOracle := strset.Difference(testSet, oracleSet).List() - return fmt.Errorf("row count differ (test has %d rows, oracle has %d rows, test is missing rows: %s, oracle is missing rows: %s)", - len(testRows), len(oracleRows), missingInTest, missingInOracle) - } - if reflect.DeepEqual(testRows, oracleRows) { + + diff := comp.GetCompareInfo(&testRows, &oracleRows, detailedDiff) + if diff.Len() == 0 { return nil } - if !detailedDiff { - return fmt.Errorf("test and oracle store have difference, detailed information will be at last attempt") - } - sort.SliceStable(testRows, func(i, j int) bool { - return lt(testRows[i], testRows[j]) - }) - sort.SliceStable(oracleRows, func(i, j int) bool { - return lt(oracleRows[i], oracleRows[j]) - }) - for i, oracleRow := range oracleRows { - testRow := testRows[i] - cmp.AllowUnexported() - diff := cmp.Diff(oracleRow, testRow, - cmpopts.SortMaps(func(x, y *inf.Dec) bool { - return x.Cmp(y) < 0 - }), - cmp.Comparer(func(x, y *inf.Dec) bool { - return x.Cmp(y) == 0 - }), cmp.Comparer(func(x, y *big.Int) bool { - return x.Cmp(y) == 0 - })) - if diff != "" { - return fmt.Errorf("rows differ (-%v +%v): %v", oracleRow, testRow, diff) - } - } - return nil + return errors.Wrap(errorResponseDiffer, diff.String()) } func (ds delegatingStore) Close() (err error) { diff --git a/pkg/typedef/types.go b/pkg/typedef/types.go index 8ac6f2e6..8f06aeb4 100644 --- a/pkg/typedef/types.go +++ b/pkg/typedef/types.go @@ -118,6 +118,10 @@ var goCQLTypeMap = map[gocql.Type]gocql.TypeInfo{ gocql.TypeCounter: gocql.NewNativeType(GoCQLProtoVersion4, gocql.TypeCounter, ""), } +func GetGoCQLTypeMap() map[gocql.Type]gocql.TypeInfo { + return goCQLTypeMap +} + type MapType struct { ComplexType string `json:"complex_type"` KeyType SimpleType `json:"key_type"` diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index ed7610b0..152c410e 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -103,3 +103,19 @@ func UUIDFromTime(rnd *rand.Rand) string { } return gocql.UUIDFromTime(RandDate(rnd)).String() } + +func RandBytes(rnd *rand.Rand, lenBytes int) []byte { + out := make([]byte, lenBytes) + for idx := range out { + out[idx] = byte(rnd.Intn(256)) + } + return out +} + +func RandSliceBytes(rnd *rand.Rand, count, lenBytes int) [][]byte { + out := make([][]byte, count) + for idx := range out { + out[idx] = RandBytes(rnd, lenBytes) + } + return out +}