Skip to content

Commit

Permalink
fix(generators): make schema generator use schema seed
Browse files Browse the repository at this point in the history
  • Loading branch information
Dmitry Kropachev committed Jul 2, 2023
1 parent ec40788 commit d9160c1
Show file tree
Hide file tree
Showing 8 changed files with 79 additions and 104 deletions.
10 changes: 7 additions & 3 deletions cmd/gemini/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ var (
outFileArg string
concurrency uint64
seed uint64
schemaSeed uint64
dropSchema bool
verbose bool
mode string
Expand Down Expand Up @@ -145,6 +146,8 @@ func run(_ *cobra.Command, _ []string) error {
globalStatus := status.NewGlobalStatus(1000)
defer utils.IgnoreError(logger.Sync)

rand.Seed(seed)

cons, err := gocql.ParseConsistencyWrapper(consistency)
if err != nil {
logger.Error("Unable parse consistency, error=%s. Falling back on Quorum", zap.Error(err))
Expand Down Expand Up @@ -198,7 +201,7 @@ func run(_ *cobra.Command, _ []string) error {
return errors.Wrap(err, "cannot create schema")
}
} else {
schema = generators.GenSchema(schemaConfig)
schema = generators.GenSchema(schemaConfig, schemaSeed)
}

jsonSchema, _ := json.MarshalIndent(schema, "", " ")
Expand Down Expand Up @@ -466,7 +469,8 @@ func init() {
rootCmd.Flags().StringVarP(&schemaFile, "schema", "", "", "Schema JSON config file")
rootCmd.Flags().StringVarP(&mode, "mode", "m", jobs.MixedMode, "Query operation mode. Mode options: write, read, mixed (default)")
rootCmd.Flags().Uint64VarP(&concurrency, "concurrency", "c", 10, "Number of threads per table to run concurrently")
rootCmd.Flags().Uint64VarP(&seed, "seed", "s", 1, "PRNG seed value")
rootCmd.Flags().Uint64VarP(&seed, "seed", "s", 1, "Statement seed value")
rootCmd.Flags().Uint64VarP(&schemaSeed, "schema-seed", "", 1, "Schema seed value")
rootCmd.Flags().BoolVarP(&dropSchema, "drop-schema", "d", false, "Drop schema before starting tests run")
rootCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Verbose output during test run")
rootCmd.Flags().BoolVarP(&failFast, "fail-fast", "f", false, "Stop on the first failure")
Expand Down Expand Up @@ -533,8 +537,8 @@ func init() {
func printSetup() error {
tw := new(tabwriter.Writer)
tw.Init(os.Stdout, 0, 8, 2, '\t', tabwriter.AlignRight)
rand.Seed(seed)
fmt.Fprintf(tw, "Seed:\t%d\n", seed)
fmt.Fprintf(tw, "Schema seed:\t%d\n", schemaSeed)
fmt.Fprintf(tw, "Maximum duration:\t%s\n", duration)
fmt.Fprintf(tw, "Warmup duration:\t%s\n", warmup)
fmt.Fprintf(tw, "Concurrency:\t%d\n", concurrency)
Expand Down
68 changes: 34 additions & 34 deletions pkg/generators/column_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,51 +26,51 @@ func GenColumnName(prefix string, idx int) string {
return fmt.Sprintf("%s%d", prefix, idx)
}

func GenColumnType(numColumns int, sc *typedef.SchemaConfig) typedef.Type {
n := rand.Intn(numColumns + 5)
func GenColumnType(numColumns int, sc *typedef.SchemaConfig, r *rand.Rand) typedef.Type {
n := r.Intn(numColumns + 5)
switch n {
case numColumns:
return GenTupleType(sc)
return GenTupleType(sc, r)
case numColumns + 1:
return GenUDTType(sc)
return GenUDTType(sc, r)
case numColumns + 2:
return GenSetType(sc)
return GenSetType(sc, r)
case numColumns + 3:
return GenListType(sc)
return GenListType(sc, r)
case numColumns + 4:
return GenMapType(sc)
return GenMapType(sc, r)
default:
return GenSimpleType(sc)
return GenSimpleType(sc, r)
}
}

func GenSimpleType(_ *typedef.SchemaConfig) typedef.SimpleType {
return typedef.AllTypes[rand.Intn(len(typedef.AllTypes))]
func GenSimpleType(_ *typedef.SchemaConfig, r *rand.Rand) typedef.SimpleType {
return typedef.AllTypes[r.Intn(len(typedef.AllTypes))]
}

func GenTupleType(sc *typedef.SchemaConfig) typedef.Type {
n := rand.Intn(sc.MaxTupleParts)
func GenTupleType(sc *typedef.SchemaConfig, r *rand.Rand) typedef.Type {
n := r.Intn(sc.MaxTupleParts)
if n < 2 {
n = 2
}
typeList := make([]typedef.SimpleType, n)
for i := 0; i < n; i++ {
typeList[i] = GenSimpleType(sc)
typeList[i] = GenSimpleType(sc, r)
}
return &typedef.TupleType{
ComplexType: typedef.TYPE_TUPLE,
ValueTypes: typeList,
Frozen: rand.Uint32()%2 == 0,
Frozen: r.Uint32()%2 == 0,
}
}

func GenUDTType(sc *typedef.SchemaConfig) *typedef.UDTType {
udtNum := rand.Uint32()
func GenUDTType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.UDTType {
udtNum := r.Uint32()
typeName := fmt.Sprintf("udt_%d", udtNum)
ts := make(map[string]typedef.SimpleType)

for i := 0; i < rand.Intn(sc.MaxUDTParts)+1; i++ {
ts[typeName+fmt.Sprintf("_%d", i)] = GenSimpleType(sc)
for i := 0; i < r.Intn(sc.MaxUDTParts)+1; i++ {
ts[typeName+fmt.Sprintf("_%d", i)] = GenSimpleType(sc, r)
}

return &typedef.UDTType{
Expand All @@ -81,51 +81,51 @@ func GenUDTType(sc *typedef.SchemaConfig) *typedef.UDTType {
}
}

func GenSetType(sc *typedef.SchemaConfig) *typedef.BagType {
return genBagType(typedef.TYPE_SET, sc)
func GenSetType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType {
return genBagType(typedef.TYPE_SET, sc, r)
}

func GenListType(sc *typedef.SchemaConfig) *typedef.BagType {
return genBagType(typedef.TYPE_LIST, sc)
func GenListType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType {
return genBagType(typedef.TYPE_LIST, sc, r)
}

func genBagType(kind string, sc *typedef.SchemaConfig) *typedef.BagType {
func genBagType(kind string, sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType {
var t typedef.SimpleType
for {
t = GenSimpleType(sc)
t = GenSimpleType(sc, r)
if t != typedef.TYPE_DURATION {
break
}
}
return &typedef.BagType{
ComplexType: kind,
ValueType: t,
Frozen: rand.Uint32()%2 == 0,
Frozen: r.Uint32()%2 == 0,
}
}

func GenMapType(sc *typedef.SchemaConfig) *typedef.MapType {
t := GenSimpleType(sc)
func GenMapType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.MapType {
t := GenSimpleType(sc, r)
for {
if _, ok := typedef.TypesMapKeyBlacklist[t]; !ok {
break
}
t = GenSimpleType(sc)
t = GenSimpleType(sc, r)
}
return &typedef.MapType{
ComplexType: typedef.TYPE_MAP,
KeyType: t,
ValueType: GenSimpleType(sc),
Frozen: rand.Uint32()%2 == 0,
ValueType: GenSimpleType(sc, r),
Frozen: r.Uint32()%2 == 0,
}
}

func GenPartitionKeyColumnType() typedef.Type {
return typedef.PartitionKeyTypes[rand.Intn(len(typedef.PartitionKeyTypes))]
func GenPartitionKeyColumnType(r *rand.Rand) typedef.Type {
return typedef.PartitionKeyTypes[r.Intn(len(typedef.PartitionKeyTypes))]
}

func GenPrimaryKeyColumnType() typedef.Type {
return typedef.PkTypes[rand.Intn(len(typedef.PkTypes))]
func GenPrimaryKeyColumnType(r *rand.Rand) typedef.Type {
return typedef.PkTypes[r.Intn(len(typedef.PkTypes))]
}

func GenIndexName(prefix string, idx int) string {
Expand Down
31 changes: 17 additions & 14 deletions pkg/generators/statement_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,35 +18,38 @@ import (
"fmt"
"strings"

"golang.org/x/exp/rand"

"github.com/scylladb/gemini/pkg/builders"
"github.com/scylladb/gemini/pkg/typedef"
"github.com/scylladb/gemini/pkg/utils"
)

func GenSchema(sc typedef.SchemaConfig) *typedef.Schema {
func GenSchema(sc typedef.SchemaConfig, seed uint64) *typedef.Schema {
r := rand.New(rand.NewSource(seed))
builder := builders.NewSchemaBuilder()
keyspace := typedef.Keyspace{
Name: "ks1",
Replication: sc.ReplicationStrategy,
OracleReplication: sc.OracleReplicationStrategy,
}
builder.Keyspace(keyspace)
numTables := utils.RandInt(1, sc.GetMaxTables())
numTables := utils.RandInt2(r, 1, sc.GetMaxTables())
for i := 0; i < numTables; i++ {
table := genTable(sc, fmt.Sprintf("table%d", i+1))
table := genTable(sc, fmt.Sprintf("table%d", i+1), r)
builder.Table(table)
}
return builder.Build()
}

func genTable(sc typedef.SchemaConfig, tableName string) *typedef.Table {
partitionKeys := make(typedef.Columns, utils.RandInt(sc.GetMinPartitionKeys(), sc.GetMaxPartitionKeys()))
func genTable(sc typedef.SchemaConfig, tableName string, r *rand.Rand) *typedef.Table {
partitionKeys := make(typedef.Columns, utils.RandInt2(r, sc.GetMinPartitionKeys(), sc.GetMaxPartitionKeys()))
for i := 0; i < len(partitionKeys); i++ {
partitionKeys[i] = &typedef.ColumnDef{Name: GenColumnName("pk", i), Type: GenPartitionKeyColumnType()}
partitionKeys[i] = &typedef.ColumnDef{Name: GenColumnName("pk", i), Type: GenPartitionKeyColumnType(r)}
}
clusteringKeys := make(typedef.Columns, utils.RandInt(sc.GetMinClusteringKeys(), sc.GetMaxClusteringKeys()))
clusteringKeys := make(typedef.Columns, utils.RandInt2(r, sc.GetMinClusteringKeys(), sc.GetMaxClusteringKeys()))
for i := 0; i < len(clusteringKeys); i++ {
clusteringKeys[i] = &typedef.ColumnDef{Name: GenColumnName("ck", i), Type: GenPrimaryKeyColumnType()}
clusteringKeys[i] = &typedef.ColumnDef{Name: GenColumnName("ck", i), Type: GenPrimaryKeyColumnType(r)}
}
table := typedef.Table{
Name: tableName,
Expand All @@ -70,21 +73,21 @@ func genTable(sc typedef.SchemaConfig, tableName string) *typedef.Table {
}
return &table
}
columns := make(typedef.Columns, utils.RandInt(sc.GetMinColumns(), sc.GetMaxColumns()))
columns := make(typedef.Columns, utils.RandInt2(r, sc.GetMinColumns(), sc.GetMaxColumns()))
for i := 0; i < len(columns); i++ {
columns[i] = &typedef.ColumnDef{Name: GenColumnName("col", i), Type: GenColumnType(len(columns), &sc)}
columns[i] = &typedef.ColumnDef{Name: GenColumnName("col", i), Type: GenColumnType(len(columns), &sc, r)}
}
table.Columns = columns

var indexes []typedef.IndexDef
if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(columns) > 0 {
indexes = CreateIndexesForColumn(&table, utils.RandInt(1, len(columns)))
indexes = CreateIndexesForColumn(&table, utils.RandInt2(r, 1, len(columns)))
}
table.Indexes = indexes

var mvs []typedef.MaterializedView
if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(clusteringKeys) > 0 {
mvs = CreateMaterializedViews(columns, table.Name, partitionKeys, clusteringKeys)
mvs = CreateMaterializedViews(columns, table.Name, partitionKeys, clusteringKeys, r)
}

table.MaterializedViews = mvs
Expand Down Expand Up @@ -142,12 +145,12 @@ func GetDropSchema(s *typedef.Schema) []string {
}
}

func CreateMaterializedViews(c typedef.Columns, tableName string, partitionKeys, clusteringKeys typedef.Columns) []typedef.MaterializedView {
func CreateMaterializedViews(c typedef.Columns, tableName string, partitionKeys, clusteringKeys typedef.Columns, r *rand.Rand) []typedef.MaterializedView {
validColumns := c.ValidColumnsForPrimaryKey()
var mvs []typedef.MaterializedView
numMvs := 1
for i := 0; i < numMvs; i++ {
col := validColumns.Random()
col := validColumns.Random(r)
if col == nil {
fmt.Printf("unable to generate valid columns for materialized view")
continue
Expand Down
38 changes: 5 additions & 33 deletions pkg/jobs/gen_ddl_stmt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"fmt"
"strings"

"github.com/pkg/errors"
"golang.org/x/exp/rand"

"github.com/scylladb/gemini/pkg/builders"
Expand All @@ -36,9 +35,12 @@ func GenDDLStmt(s *typedef.Schema, t *typedef.Table, r *rand.Rand, _ *typedef.Pa
// case 0: // Alter column not supported in Cassandra from 3.0.11
// return t.alterColumn(s.Keyspace.Name)
case 2:
return genDropColumnStmt(t, s.Keyspace.Name, validCols.Random())
return genDropColumnStmt(t, s.Keyspace.Name, validCols.Random(r))
default:
column := typedef.ColumnDef{Name: generators.GenColumnName("col", len(t.Columns)+1), Type: generators.GenColumnType(len(t.Columns)+1, sc)}
column := typedef.ColumnDef{
Name: generators.GenColumnName("col", len(t.Columns)+1),
Type: generators.GenColumnType(len(t.Columns)+1, sc, r),
}
return genAddColumnStmt(t, s.Keyspace.Name, &column)
}
}
Expand Down Expand Up @@ -82,36 +84,6 @@ func genAddColumnStmt(t *typedef.Table, keyspace string, column *typedef.ColumnD
}, nil
}

//nolint:unused
func alterColumn(t *typedef.Table, keyspace string) ([]*typedef.Stmt, func(), error) {
var stmts []*typedef.Stmt
idx := rand.Intn(len(t.Columns))
column := t.Columns[idx]
oldType, isSimpleType := column.Type.(typedef.SimpleType)
if !isSimpleType {
return nil, func() {}, errors.Errorf("complex type=%s cannot be altered", column.Name)
}
compatTypes := typedef.CompatibleColumnTypes[oldType]
if len(compatTypes) == 0 {
return nil, func() {}, errors.Errorf("simple type=%s has no compatible coltypes so it cannot be altered", column.Name)
}
newType := compatTypes.Random()
newColumn := typedef.ColumnDef{Name: column.Name, Type: newType}
stmt := "ALTER TABLE " + keyspace + "." + t.Name + " ALTER " + column.Name + " TYPE " + column.Type.CQLDef()
stmts = append(stmts, &typedef.Stmt{
StmtCache: &typedef.StmtCache{
Query: &builders.AlterTableBuilder{
Stmt: stmt,
},
QueryType: typedef.AlterColumnStatementType,
},
})
return stmts, func() {
t.Columns[idx] = &newColumn
t.ResetQueryCache()
}, nil
}

func genDropColumnStmt(t *typedef.Table, keyspace string, column *typedef.ColumnDef) (*typedef.Stmts, error) {
var stmts []*typedef.Stmt

Expand Down
4 changes: 2 additions & 2 deletions pkg/typedef/columns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,8 @@ func (c Columns) ValidColumnsForPrimaryKey() Columns {
return validCols
}

func (c Columns) Random() *ColumnDef {
return c[rand.Intn(len(c))]
func (c Columns) Random(r *rand.Rand) *ColumnDef {
return c[r.Intn(len(c))]
}

func (c Columns) LenValues() int {
Expand Down
Loading

0 comments on commit d9160c1

Please sign in to comment.