From e403c63943901d04e99269516396f898204503f9 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Tue, 15 Feb 2022 21:57:34 +0530 Subject: [PATCH 01/20] Benchmark performance for HarbourBridge migration --- cmd/cmd.go | 4 +- cmd/data.go | 8 +- cmd/schema.go | 6 +- cmd/schema_and_data.go | 18 ++-- conversion/conversion.go | 4 +- internal/convert.go | 34 ++++---- internal/report.go | 15 ++++ .../cleanup_resource/cleanup_resource.go | 37 ++++++++ .../populate_database/populate_database.go | 86 +++++++++++++++++++ performance/setup.sh | 53 ++++++++++++ performance/util.go | 52 +++++++++++ 11 files changed, 289 insertions(+), 28 deletions(-) create mode 100644 performance/cleanup_resource/cleanup_resource.go create mode 100644 performance/populate_database/populate_database.go create mode 100644 performance/setup.sh create mode 100644 performance/util.go diff --git a/cmd/cmd.go b/cmd/cmd.go index 287cb76f1..ce44ffac5 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -36,6 +36,8 @@ var ( sessionFile = "session.json" ) +const defaultWriteLimit = 40 + // CommandLine provides the core processing for HarbourBridge when run as a command-line tool. // It performs the following steps: // 1. Run schema conversion (if dataOnly is set to false) @@ -100,7 +102,7 @@ func CommandLine(ctx context.Context, driver, targetDb, dbURI string, dataOnly, // We pass an empty string to the sqlConnectionStr parameter as this is the legacy codepath, // which reads the environment variables and constructs the string later on. - bw, err := conversion.DataConv(ctx, sourceProfile, targetProfile, ioHelper, client, conv, dataOnly) + bw, err := conversion.DataConv(ctx, sourceProfile, targetProfile, ioHelper, client, conv, dataOnly, defaultWriteLimit) if err != nil { return fmt.Errorf("can't finish data conversion for db %s: %v", dbURI, err) } diff --git a/cmd/data.go b/cmd/data.go index 7de3f476b..3c3ec09d1 100644 --- a/cmd/data.go +++ b/cmd/data.go @@ -25,6 +25,7 @@ type DataCmd struct { skipForeignKeys bool sessionJSON string filePrefix string // TODO: move filePrefix to global flags + writeLimit int64 } // Name returns the name of operation. @@ -57,6 +58,7 @@ func (cmd *DataCmd) SetFlags(f *flag.FlagSet) { f.StringVar(&cmd.targetProfile, "target-profile", "", "Flag for specifying connection profile for target database e.g., \"dialect=postgresql\"") f.BoolVar(&cmd.skipForeignKeys, "skip-foreign-keys", false, "Skip creating foreign keys after data migration is complete (ddl statements for foreign keys can still be found in the downloaded schema.ddl.txt file and the same can be applied separately)") f.StringVar(&cmd.filePrefix, "prefix", "", "File prefix for generated files") + f.Int64Var(&cmd.writeLimit, "write-limit", 40, "Write limit for writes to spanner") } func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { @@ -143,7 +145,8 @@ func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface } } - bw, err := conversion.DataConv(ctx, sourceProfile, targetProfile, &ioHelper, client, conv, true) + dataCoversionStartTime := time.Now() + bw, err := conversion.DataConv(ctx, sourceProfile, targetProfile, &ioHelper, client, conv, true, cmd.writeLimit) if err != nil { err = fmt.Errorf("can't finish data conversion for db %s: %v", dbURI, err) return subcommands.ExitFailure @@ -154,6 +157,9 @@ func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface return subcommands.ExitFailure } } + dataCoversionEndTime := time.Now() + dataCoversionDuration := dataCoversionEndTime.Sub(dataCoversionStartTime) + conv.DataConversionDuration = dataCoversionDuration banner := utils.GetBanner(now, dbURI) conversion.Report(sourceProfile.Driver, bw.DroppedRowsByTable(), ioHelper.BytesRead, banner, conv, cmd.filePrefix+reportFile, ioHelper.Out) conversion.WriteBadData(bw, conv, banner, cmd.filePrefix+badDataFile, ioHelper.Out) diff --git a/cmd/schema.go b/cmd/schema.go index 42d9a0c06..7df8843d3 100644 --- a/cmd/schema.go +++ b/cmd/schema.go @@ -99,14 +99,16 @@ func (cmd *SchemaCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interfa cmd.filePrefix = dbName + "." } + schemaConversionStartTime := time.Now() var conv *internal.Conv conv, err = conversion.SchemaConv(sourceProfile, targetProfile, &ioHelper) if err != nil { return subcommands.ExitFailure } - now := time.Now() - conversion.WriteSchemaFile(conv, now, cmd.filePrefix+schemaFile, ioHelper.Out) + schemaCoversionEndTime := time.Now() + conv.SchemaConversionDuration = schemaCoversionEndTime.Sub(schemaConversionStartTime) + conversion.WriteSchemaFile(conv, schemaCoversionEndTime, cmd.filePrefix+schemaFile, ioHelper.Out) conversion.WriteSessionFile(conv, cmd.filePrefix+sessionFile, ioHelper.Out) conversion.Report(sourceProfile.Driver, nil, ioHelper.BytesRead, "", conv, cmd.filePrefix+reportFile, ioHelper.Out) // Cleanup hb tmp data directory. diff --git a/cmd/schema_and_data.go b/cmd/schema_and_data.go index e0b463d75..19f943f4c 100644 --- a/cmd/schema_and_data.go +++ b/cmd/schema_and_data.go @@ -24,6 +24,7 @@ type SchemaAndDataCmd struct { targetProfile string skipForeignKeys bool filePrefix string // TODO: move filePrefix to global flags + writeLimit int64 } // Name returns the name of operation. @@ -55,6 +56,7 @@ func (cmd *SchemaAndDataCmd) SetFlags(f *flag.FlagSet) { f.StringVar(&cmd.targetProfile, "target-profile", "", "Flag for specifying connection profile for target database e.g., \"dialect=postgresql\"") f.BoolVar(&cmd.skipForeignKeys, "skip-foreign-keys", false, "Skip creating foreign keys after data migration is complete (ddl statements for foreign keys can still be found in the downloaded schema.ddl.txt file and the same can be applied separately)") f.StringVar(&cmd.filePrefix, "prefix", "", "File prefix for generated files") + f.Int64Var(&cmd.writeLimit, "write-limit", 40, "Write limit for writes to spanner") } func (cmd *SchemaAndDataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { @@ -91,11 +93,11 @@ func (cmd *SchemaAndDataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ... defer ioHelper.In.Close() } - now := time.Now() + schemaConversionStartTime := time.Now() // If filePrefix not explicitly set, use dbName as prefix. if cmd.filePrefix == "" { - dbName, err := utils.GetDatabaseName(sourceProfile.Driver, now) + dbName, err := utils.GetDatabaseName(sourceProfile.Driver, schemaConversionStartTime) if err != nil { panic(fmt.Errorf("can't generate database name for prefix: %v", err)) } @@ -108,11 +110,11 @@ func (cmd *SchemaAndDataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ... panic(err) } - conversion.WriteSchemaFile(conv, now, cmd.filePrefix+schemaFile, ioHelper.Out) + conversion.WriteSchemaFile(conv, schemaConversionStartTime, cmd.filePrefix+schemaFile, ioHelper.Out) conversion.WriteSessionFile(conv, cmd.filePrefix+sessionFile, ioHelper.Out) conversion.Report(sourceProfile.Driver, nil, ioHelper.BytesRead, "", conv, cmd.filePrefix+reportFile, ioHelper.Out) - project, instance, dbName, err := targetProfile.GetResourceIds(ctx, now, sourceProfile.Driver, ioHelper.Out) + project, instance, dbName, err := targetProfile.GetResourceIds(ctx, schemaConversionStartTime, sourceProfile.Driver, ioHelper.Out) if err != nil { return subcommands.ExitUsageError } @@ -141,7 +143,9 @@ func (cmd *SchemaAndDataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ... return subcommands.ExitFailure } - bw, err := conversion.DataConv(ctx, sourceProfile, targetProfile, &ioHelper, client, conv, true) + schemaCoversionEndTime := time.Now() + conv.SchemaConversionDuration = schemaCoversionEndTime.Sub(schemaConversionStartTime) + bw, err := conversion.DataConv(ctx, sourceProfile, targetProfile, &ioHelper, client, conv, true, cmd.writeLimit) if err != nil { err = fmt.Errorf("can't finish data conversion for db %s: %v", dbURI, err) return subcommands.ExitFailure @@ -152,7 +156,9 @@ func (cmd *SchemaAndDataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ... return subcommands.ExitFailure } } - banner := utils.GetBanner(now, dbURI) + dataCoversionEndTime := time.Now() + conv.DataConversionDuration = dataCoversionEndTime.Sub(schemaCoversionEndTime) + banner := utils.GetBanner(schemaConversionStartTime, dbURI) conversion.Report(sourceProfile.Driver, bw.DroppedRowsByTable(), ioHelper.BytesRead, banner, conv, cmd.filePrefix+reportFile, ioHelper.Out) conversion.WriteBadData(bw, conv, banner, cmd.filePrefix+badDataFile, ioHelper.Out) // Cleanup hb tmp data directory. diff --git a/conversion/conversion.go b/conversion/conversion.go index fde16b1a0..d7981751f 100644 --- a/conversion/conversion.go +++ b/conversion/conversion.go @@ -93,10 +93,10 @@ func SchemaConv(sourceProfile profiles.SourceProfile, targetProfile profiles.Tar // - Driver is DynamoDB or a dump file mode. // - This function is called as part of the legacy global CLI flag mode. (This string is constructed from env variables later on) // When using source-profile, the sqlConnectionStr and schemaSampleSize are constructed from the input params. -func DataConv(ctx context.Context, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile, ioHelper *utils.IOStreams, client *sp.Client, conv *internal.Conv, dataOnly bool) (*writer.BatchWriter, error) { +func DataConv(ctx context.Context, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile, ioHelper *utils.IOStreams, client *sp.Client, conv *internal.Conv, dataOnly bool, writeLimit int64) (*writer.BatchWriter, error) { config := writer.BatchWriterConfig{ BytesLimit: 100 * 1000 * 1000, - WriteLimit: 40, + WriteLimit: writeLimit, RetryLimit: 1000, Verbose: internal.Verbose(), } diff --git a/internal/convert.go b/internal/convert.go index d8071f2f9..e08bad35f 100644 --- a/internal/convert.go +++ b/internal/convert.go @@ -24,22 +24,24 @@ import ( // Conv contains all schema and data conversion state. type Conv struct { - mode mode // Schema mode or data mode. - SpSchema ddl.Schema // Maps Spanner table name to Spanner schema. - SyntheticPKeys map[string]SyntheticPKey // Maps Spanner table name to synthetic primary key (if needed). - SrcSchema map[string]schema.Table // Maps source-DB table name to schema information. - Issues map[string]map[string][]SchemaIssue // Maps source-DB table/col to list of schema conversion issues. - ToSpanner map[string]NameAndCols // Maps from source-DB table name to Spanner name and column mapping. - ToSource map[string]NameAndCols // Maps from Spanner table name to source-DB table name and column mapping. - UsedNames map[string]bool // Map storing the names that are already assigned to tables, indices or foreign key contraints. - dataSink func(table string, cols []string, values []interface{}) - DataFlush func() `json:"-"` // Data flush is used to flush out remaining writes and wait for them to complete. - Location *time.Location // Timezone (for timestamp conversion). - sampleBadRows rowSamples // Rows that generated errors during conversion. - Stats stats - TimezoneOffset string // Timezone offset for timestamp conversion. - TargetDb string // The target database to which HarbourBridge is writing. - UniquePKey map[string][]string // Maps Spanner table name to unique column name being used as primary key (if needed). + mode mode // Schema mode or data mode. + SpSchema ddl.Schema // Maps Spanner table name to Spanner schema. + SyntheticPKeys map[string]SyntheticPKey // Maps Spanner table name to synthetic primary key (if needed). + SrcSchema map[string]schema.Table // Maps source-DB table name to schema information. + Issues map[string]map[string][]SchemaIssue // Maps source-DB table/col to list of schema conversion issues. + ToSpanner map[string]NameAndCols // Maps from source-DB table name to Spanner name and column mapping. + ToSource map[string]NameAndCols // Maps from Spanner table name to source-DB table name and column mapping. + UsedNames map[string]bool // Map storing the names that are already assigned to tables, indices or foreign key contraints. + dataSink func(table string, cols []string, values []interface{}) + DataFlush func() `json:"-"` // Data flush is used to flush out remaining writes and wait for them to complete. + Location *time.Location // Timezone (for timestamp conversion). + sampleBadRows rowSamples // Rows that generated errors during conversion. + Stats stats + TimezoneOffset string // Timezone offset for timestamp conversion. + TargetDb string // The target database to which HarbourBridge is writing. + UniquePKey map[string][]string // Maps Spanner table name to unique column name being used as primary key (if needed). + SchemaConversionDuration time.Duration `json:"-"` + DataConversionDuration time.Duration `json:"-"` } type mode int diff --git a/internal/report.go b/internal/report.go index b3fc4f319..6e82b294b 100644 --- a/internal/report.go +++ b/internal/report.go @@ -34,6 +34,7 @@ func GenerateReport(driverName string, conv *Conv, w *bufio.Writer, badWrites ma w.WriteString(summary) ignored := IgnoredStatements(conv) w.WriteString("\n") + w.WriteString(conversionDuration(conv, w)) if len(ignored) > 0 { justifyLines(w, fmt.Sprintf("Note that the following source DB statements "+ "were detected but ignored: %s.", @@ -557,3 +558,17 @@ func writeHeading(w *bufio.Writer, s string) { s, "\n", "----------------------------\n"}, "")) } + +func conversionDuration(conv *Conv, w *bufio.Writer) string { + if conv.DataConversionDuration.Microseconds() != 0 || conv.SchemaConversionDuration.Microseconds() != 0 { + writeHeading(w, "Time duration of Conversion") + if conv.DataConversionDuration.Microseconds() != 0 && conv.SchemaConversionDuration.Microseconds() != 0 { + return fmt.Sprintf("Schema conversion duration : %s \nData conversion duration : %s \n\n", conv.SchemaConversionDuration, conv.DataConversionDuration) + } else if conv.DataConversionDuration.Microseconds() != 0 { + return fmt.Sprintf("Data conversion duration : %s \n\n", conv.DataConversionDuration) + } else if conv.SchemaConversionDuration.Microseconds() != 0 { + return fmt.Sprintf("Schema conversion duration : %s \n\n", conv.SchemaConversionDuration) + } + } + return "" +} diff --git a/performance/cleanup_resource/cleanup_resource.go b/performance/cleanup_resource/cleanup_resource.go new file mode 100644 index 000000000..bc642e486 --- /dev/null +++ b/performance/cleanup_resource/cleanup_resource.go @@ -0,0 +1,37 @@ +package main + +import ( + "database/sql" + "log" + "os" + + "github.com/cloudspannerecosystem/harbourbridge/performance" +) + +const ( + dbName = "testdb" +) + +func main() { + host, user, password, port := os.Getenv("MYSQLHOST"), os.Getenv("MYSQLUSER"), os.Getenv("MYSQLPWD"), os.Getenv("MYSQLPORT") + connString := performance.GetMYSQLConnectionStr(host, port, user, password, "") + db, err := sql.Open("mysql", connString) + if err != nil { + panic(err) + } + defer db.Close() + + // Drop MySQL database. + _, err = db.Exec("DROP DATABASE IF EXISTS " + dbName) + if err != nil { + panic(err) + } + + // Delete the csv file. + if _, err := os.Stat("records.csv"); err == nil { + err = os.Remove("records.csv") + if err != nil { + log.Fatalln("failed to delete file", err) + } + } +} diff --git a/performance/populate_database/populate_database.go b/performance/populate_database/populate_database.go new file mode 100644 index 000000000..4de2d5674 --- /dev/null +++ b/performance/populate_database/populate_database.go @@ -0,0 +1,86 @@ +package main + +import ( + "database/sql" + "encoding/csv" + "flag" + "log" + "os" + "strconv" + + "github.com/cloudspannerecosystem/harbourbridge/performance" + "github.com/go-sql-driver/mysql" +) + +const ( + dbName = "testdb" +) + +var ( + recordCount int + multipleTableDb bool +) + +func main() { + flag.IntVar(&recordCount, "record-count", 10000, "record-count: Number of rows to add") + flag.BoolVar(&multipleTableDb, "multiple-table-db", false, "multiple-table-db: it is set to true for populating multiple table database") + flag.Parse() + file, err := os.Create("records.csv") + defer file.Close() + if err != nil { + log.Fatalln("failed to open file", err) + } + w := csv.NewWriter(file) + defer w.Flush() + + host, user, password, port := os.Getenv("MYSQLHOST"), os.Getenv("MYSQLUSER"), os.Getenv("MYSQLPWD"), os.Getenv("MYSQLPORT") + connString := performance.GetMYSQLConnectionStr(host, port, user, password, "") + db, err := sql.Open("mysql", connString) + if err != nil { + panic(err) + } + defer db.Close() + + _, err = db.Exec("CREATE DATABASE IF NOT EXISTS " + dbName) + if err != nil { + panic(err) + } + + _, err = db.Exec("USE " + dbName) + if err != nil { + panic(err) + } + if !multipleTableDb { + // Writing data to csv file. + var data [][]string + for i := 1; i <= recordCount; i++ { + row := []string{performance.RandomString(5), performance.RandomString(10), performance.RandomString(10), performance.RandomString(50), performance.RandomDate(), + strconv.FormatBool(performance.RandomBool()), strconv.FormatFloat(performance.RandomFloat(150, 200), 'E', -1, 64), strconv.Itoa(int(performance.RandomInt(1000, 100000))), performance.CurrentTimestamp()} + data = append(data, row) + } + w.WriteAll(data) + + // MySQL table creation. + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS employee(employee_id varchar(50) PRIMARY KEY, first_name varchar(50) NOT NULL, + last_name varchar(50), address varchar(100), dob DATE NOT NULL, is_manager bool NOT NULL, height_in_cm float(4,1) NOT NULL, + salary integer NOT NULL, last_updated_time TIMESTAMP NOT NULL)`) + if err != nil { + panic(err) + } + connString = performance.GetMYSQLConnectionStr(host, port, user, password, "testdb") + if err != nil { + panic(err) + } + db, err = sql.Open("mysql", connString) + if err != nil { + panic(err) + } + + // Loading data into MySQL database from the locally generated csv file. + mysql.RegisterLocalFile("records.csv") + _, err = db.Exec("LOAD DATA LOCAL INFILE 'records.csv' INTO TABLE employee FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n'; ") + if err != nil { + panic(err.Error()) + } + } +} diff --git a/performance/setup.sh b/performance/setup.sh new file mode 100644 index 000000000..bf2e1fe33 --- /dev/null +++ b/performance/setup.sh @@ -0,0 +1,53 @@ +#!/bin/sh +if type mysql >/dev/null 2>&1; then + echo "MySQL present." + export MYSQLHOST=localhost + export MYSQLUSER=root + export MYSQLPORT=3306 + #update MySQL password before running the benchmark + export MYSQLPWD= + gcloud auth application-default login + #benchmark for single table database + for insertRecordCount in 250000 250000 1000000 + do + go run ./performance/populate_database/populate_database.go -record-count $insertRecordCount + echo "$insertRecordCount records inserted." + for nodes in 1 3 5 + do + gcloud spanner instances create new-test-instance --config=regional-us-central1 \ + --description="New test Instance" --nodes=$nodes + gcloud config set spanner/instance new-test-instance + echo "Spanner instance created with $nodes nodes" + for writeLimit in 20 40 60 80 100 + do + echo "Iteration $threads" + #update source profile password before running the benchmark + go run main.go schema-and-data -source=mysql -source-profile='host=localhost,user=root,db_name=testdb,password=' -target-profile='instance=new-test-instance,dbname=testdb' -write-limit $writeLimit + done + yes | gcloud spanner instances delete test-instance + done + done + go run ./performance/drop_database/drop_database.go + #benchmark for multiple table database + for insertRecordCount in 250000 250000 1000000 + do + go run ./performance/populate_database/populate_database.go -record-count $insertRecordCount -multiple-table-db + echo "$insertRecordCount records inserted." + for nodes in 1 3 5 + do + gcloud spanner instances create new-test-instance --config=regional-us-central1 \ + --description="New test Instance" --nodes=$nodes + gcloud config set spanner/instance new-test-instance + for threads in 20 40 60 80 100 + do + echo "Iteration $threads" + #update source profile password before running the benchmark + go run main.go schema-and-data -source=mysql -source-profile='host=localhost,user=root,db_name=testdb,password=' -target-profile='instance=new-test-instance,dbname=testdb' -write-limit $writeLimit + done + yes | gcloud spanner instances delete test-instance + done + done + go run ./performance/drop_database/drop_database.go +else + echo "MySQL not present. Please install it from: https://dev.mysql.com/doc/mysql-shell/8.0/en/mysql-shell-install.html" +fi \ No newline at end of file diff --git a/performance/util.go b/performance/util.go new file mode 100644 index 000000000..cf8b8fd5c --- /dev/null +++ b/performance/util.go @@ -0,0 +1,52 @@ +package performance + +import ( + "fmt" + "math/rand" + "strings" + "time" +) + +func RandomInt(min, max int64) int64 { + return min + rand.Int63n(max-min+1) +} + +func RandomFloat(min, max int64) float64 { + return float64(min) + rand.Float64()*float64(max-min) +} + +const alphabet = "abcdefghijklmnopqrstuvwxyz" + +func RandomString(n int) string { + var sb strings.Builder + k := len(alphabet) + + for i := 0; i < n; i++ { + c := alphabet[rand.Intn(k)] + sb.WriteByte(c) + } + + return sb.String() +} + +func RandomDate() string { + min := time.Date(1970, 1, 0, 0, 0, 0, 0, time.UTC).Unix() + max := time.Date(2010, 1, 0, 0, 0, 0, 0, time.UTC).Unix() + delta := max - min + + sec := rand.Int63n(delta) + min + return time.Unix(sec, 0).Format("2006-01-02") +} + +func RandomBool() bool { + i := RandomInt(0, 10) + return i%2 == 0 +} + +func CurrentTimestamp() string { + return time.Now().Format("2006-01-02 15:04:05") +} + +func GetMYSQLConnectionStr(server, port, user, password, dbName string) string { + return fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", user, password, server, port, dbName) +} From 66908f44e051f13530f84ea5a2a691aa03b53162 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Tue, 15 Feb 2022 22:14:10 +0530 Subject: [PATCH 02/20] minor fixes --- performance/cleanup_resource/cleanup_resource.go | 2 ++ performance/setup.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/performance/cleanup_resource/cleanup_resource.go b/performance/cleanup_resource/cleanup_resource.go index bc642e486..2e2e3212e 100644 --- a/performance/cleanup_resource/cleanup_resource.go +++ b/performance/cleanup_resource/cleanup_resource.go @@ -5,6 +5,8 @@ import ( "log" "os" + _ "github.com/go-sql-driver/mysql" + "github.com/cloudspannerecosystem/harbourbridge/performance" ) diff --git a/performance/setup.sh b/performance/setup.sh index bf2e1fe33..4e7664e01 100644 --- a/performance/setup.sh +++ b/performance/setup.sh @@ -27,7 +27,7 @@ if type mysql >/dev/null 2>&1; then yes | gcloud spanner instances delete test-instance done done - go run ./performance/drop_database/drop_database.go + go run ./performance/cleanup_resource/cleanup_resource.go #benchmark for multiple table database for insertRecordCount in 250000 250000 1000000 do @@ -47,7 +47,7 @@ if type mysql >/dev/null 2>&1; then yes | gcloud spanner instances delete test-instance done done - go run ./performance/drop_database/drop_database.go + go run ./performance/cleanup_resource/cleanup_resource.go else echo "MySQL not present. Please install it from: https://dev.mysql.com/doc/mysql-shell/8.0/en/mysql-shell-install.html" fi \ No newline at end of file From 5e694702c7bd075b2797b816fd17f96e9b8527a1 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Tue, 15 Feb 2022 22:16:20 +0530 Subject: [PATCH 03/20] minor fixes --- internal/convert.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/convert.go b/internal/convert.go index e08bad35f..f2f62c9bc 100644 --- a/internal/convert.go +++ b/internal/convert.go @@ -40,8 +40,8 @@ type Conv struct { TimezoneOffset string // Timezone offset for timestamp conversion. TargetDb string // The target database to which HarbourBridge is writing. UniquePKey map[string][]string // Maps Spanner table name to unique column name being used as primary key (if needed). - SchemaConversionDuration time.Duration `json:"-"` - DataConversionDuration time.Duration `json:"-"` + SchemaConversionDuration time.Duration `json:"-"` // Duration of schema conversion. + DataConversionDuration time.Duration `json:"-"` // Duration of data conversion. } type mode int From 79d35921553ba4b2c434767d3dbcd0db356823f3 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Fri, 18 Feb 2022 13:01:41 +0530 Subject: [PATCH 04/20] addressing comments --- cmd/cmd.go | 4 ++-- cmd/data.go | 2 +- cmd/schema_and_data.go | 2 +- internal/report.go | 15 ++++++++------- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index ce44ffac5..a577ca5dd 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -36,7 +36,7 @@ var ( sessionFile = "session.json" ) -const defaultWriteLimit = 40 +const defaultWritersLimit = 40 // CommandLine provides the core processing for HarbourBridge when run as a command-line tool. // It performs the following steps: @@ -102,7 +102,7 @@ func CommandLine(ctx context.Context, driver, targetDb, dbURI string, dataOnly, // We pass an empty string to the sqlConnectionStr parameter as this is the legacy codepath, // which reads the environment variables and constructs the string later on. - bw, err := conversion.DataConv(ctx, sourceProfile, targetProfile, ioHelper, client, conv, dataOnly, defaultWriteLimit) + bw, err := conversion.DataConv(ctx, sourceProfile, targetProfile, ioHelper, client, conv, dataOnly, defaultWritersLimit) if err != nil { return fmt.Errorf("can't finish data conversion for db %s: %v", dbURI, err) } diff --git a/cmd/data.go b/cmd/data.go index 3c3ec09d1..a0118fc0d 100644 --- a/cmd/data.go +++ b/cmd/data.go @@ -58,7 +58,7 @@ func (cmd *DataCmd) SetFlags(f *flag.FlagSet) { f.StringVar(&cmd.targetProfile, "target-profile", "", "Flag for specifying connection profile for target database e.g., \"dialect=postgresql\"") f.BoolVar(&cmd.skipForeignKeys, "skip-foreign-keys", false, "Skip creating foreign keys after data migration is complete (ddl statements for foreign keys can still be found in the downloaded schema.ddl.txt file and the same can be applied separately)") f.StringVar(&cmd.filePrefix, "prefix", "", "File prefix for generated files") - f.Int64Var(&cmd.writeLimit, "write-limit", 40, "Write limit for writes to spanner") + f.Int64Var(&cmd.writeLimit, "write-limit", defaultWritersLimit, "Write limit for writes to spanner") } func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { diff --git a/cmd/schema_and_data.go b/cmd/schema_and_data.go index 19f943f4c..eb27c532c 100644 --- a/cmd/schema_and_data.go +++ b/cmd/schema_and_data.go @@ -56,7 +56,7 @@ func (cmd *SchemaAndDataCmd) SetFlags(f *flag.FlagSet) { f.StringVar(&cmd.targetProfile, "target-profile", "", "Flag for specifying connection profile for target database e.g., \"dialect=postgresql\"") f.BoolVar(&cmd.skipForeignKeys, "skip-foreign-keys", false, "Skip creating foreign keys after data migration is complete (ddl statements for foreign keys can still be found in the downloaded schema.ddl.txt file and the same can be applied separately)") f.StringVar(&cmd.filePrefix, "prefix", "", "File prefix for generated files") - f.Int64Var(&cmd.writeLimit, "write-limit", 40, "Write limit for writes to spanner") + f.Int64Var(&cmd.writeLimit, "write-limit", defaultWritersLimit, "Write limit for writes to spanner") } func (cmd *SchemaAndDataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { diff --git a/internal/report.go b/internal/report.go index 6e82b294b..f154b877e 100644 --- a/internal/report.go +++ b/internal/report.go @@ -560,15 +560,16 @@ func writeHeading(w *bufio.Writer, s string) { } func conversionDuration(conv *Conv, w *bufio.Writer) string { + res := "" if conv.DataConversionDuration.Microseconds() != 0 || conv.SchemaConversionDuration.Microseconds() != 0 { writeHeading(w, "Time duration of Conversion") - if conv.DataConversionDuration.Microseconds() != 0 && conv.SchemaConversionDuration.Microseconds() != 0 { - return fmt.Sprintf("Schema conversion duration : %s \nData conversion duration : %s \n\n", conv.SchemaConversionDuration, conv.DataConversionDuration) - } else if conv.DataConversionDuration.Microseconds() != 0 { - return fmt.Sprintf("Data conversion duration : %s \n\n", conv.DataConversionDuration) - } else if conv.SchemaConversionDuration.Microseconds() != 0 { - return fmt.Sprintf("Schema conversion duration : %s \n\n", conv.SchemaConversionDuration) + if conv.SchemaConversionDuration.Microseconds() != 0 { + res += fmt.Sprintf("Schema conversion duration : %s \n", conv.SchemaConversionDuration) } + if conv.DataConversionDuration.Microseconds() != 0 { + res += fmt.Sprintf("Data conversion duration : %s \n", conv.DataConversionDuration) + } + res += "\n" } - return "" + return res } From 6abd1da84962868a9100c6a369ac778c946083d3 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Sat, 19 Feb 2022 16:35:44 +0530 Subject: [PATCH 05/20] addressing comments --- performance/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/performance/setup.sh b/performance/setup.sh index 4e7664e01..43acb0a70 100644 --- a/performance/setup.sh +++ b/performance/setup.sh @@ -20,7 +20,7 @@ if type mysql >/dev/null 2>&1; then echo "Spanner instance created with $nodes nodes" for writeLimit in 20 40 60 80 100 do - echo "Iteration $threads" + echo "Write limit: $writeLimit" #update source profile password before running the benchmark go run main.go schema-and-data -source=mysql -source-profile='host=localhost,user=root,db_name=testdb,password=' -target-profile='instance=new-test-instance,dbname=testdb' -write-limit $writeLimit done @@ -40,7 +40,7 @@ if type mysql >/dev/null 2>&1; then gcloud config set spanner/instance new-test-instance for threads in 20 40 60 80 100 do - echo "Iteration $threads" + echo "Write limit: $writeLimit" #update source profile password before running the benchmark go run main.go schema-and-data -source=mysql -source-profile='host=localhost,user=root,db_name=testdb,password=' -target-profile='instance=new-test-instance,dbname=testdb' -write-limit $writeLimit done From 4a1c7140e49efed22d1a743c4a0c0f9a21f6816b Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Tue, 22 Feb 2022 10:53:13 +0530 Subject: [PATCH 06/20] send test context variable --- conversion/conversion.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conversion/conversion.go b/conversion/conversion.go index d7981751f..20cbd6dc5 100644 --- a/conversion/conversion.go +++ b/conversion/conversion.go @@ -506,7 +506,11 @@ func CreateDatabase(ctx context.Context, adminClient *database.DatabaseAdminClie req.ExtraStatements = conv.SpSchema.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: true, ForeignKeys: false, TargetDb: conv.TargetDb}) } - op, err := adminClient.CreateDatabase(ctx, req) + ctx1 := context.WithValue(ctx, "test-key", "test-value") + print("create database statment\n") + fmt.Println(ctx1.Value("test-key")) + fmt.Println(ctx.Value("test-key")) + op, err := adminClient.CreateDatabase(ctx1, req) if err != nil { return fmt.Errorf("can't build CreateDatabaseRequest: %w", utils.AnalyzeError(err, dbURI)) } From 5cca99e98f2d3d0157bd0b28497b3453a87ff3f7 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Tue, 22 Feb 2022 11:03:26 +0530 Subject: [PATCH 07/20] removing extra changes --- conversion/conversion.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/conversion/conversion.go b/conversion/conversion.go index 20cbd6dc5..d7981751f 100644 --- a/conversion/conversion.go +++ b/conversion/conversion.go @@ -506,11 +506,7 @@ func CreateDatabase(ctx context.Context, adminClient *database.DatabaseAdminClie req.ExtraStatements = conv.SpSchema.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: true, ForeignKeys: false, TargetDb: conv.TargetDb}) } - ctx1 := context.WithValue(ctx, "test-key", "test-value") - print("create database statment\n") - fmt.Println(ctx1.Value("test-key")) - fmt.Println(ctx.Value("test-key")) - op, err := adminClient.CreateDatabase(ctx1, req) + op, err := adminClient.CreateDatabase(ctx, req) if err != nil { return fmt.Errorf("can't build CreateDatabaseRequest: %w", utils.AnalyzeError(err, dbURI)) } From dadc1611e0c5cc22ed4f0e5cbeca4bbc74a53fad Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Sat, 26 Feb 2022 11:18:25 +0530 Subject: [PATCH 08/20] added proto file --- go.mod | 2 + proto/migration/migration_data.pb.go | 663 +++++++++++++++++++++++++++ proto/migration_data.proto | 71 +++ 3 files changed, 736 insertions(+) create mode 100644 proto/migration/migration_data.pb.go create mode 100644 proto/migration_data.proto diff --git a/go.mod b/go.mod index 28102ea05..0a60442e0 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/go-sql-driver/mysql v1.5.0 + github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.6 github.com/google/subcommands v1.2.0 github.com/gorilla/handlers v1.5.0 @@ -30,6 +31,7 @@ require ( golang.org/x/tools v0.1.7 // indirect google.golang.org/api v0.57.0 google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 + google.golang.org/protobuf v1.27.1 // indirect honnef.co/go/tools v0.2.1 // indirect ) diff --git a/proto/migration/migration_data.pb.go b/proto/migration/migration_data.pb.go new file mode 100644 index 000000000..051e02dc9 --- /dev/null +++ b/proto/migration/migration_data.pb.go @@ -0,0 +1,663 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.27.1 +// protoc v3.19.4 +// source: migration_data.proto + +package migration + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type MigrationData_Source int32 + +const ( + MigrationData_SOURCE_UNSPECIFIED MigrationData_Source = 0 + MigrationData_MYSQL MigrationData_Source = 1 + MigrationData_POSTGRESQL MigrationData_Source = 2 + MigrationData_DYNAMODB MigrationData_Source = 3 + MigrationData_SQL_SERVER MigrationData_Source = 4 + MigrationData_ORACLE MigrationData_Source = 5 +) + +// Enum value maps for MigrationData_Source. +var ( + MigrationData_Source_name = map[int32]string{ + 0: "SOURCE_UNSPECIFIED", + 1: "MYSQL", + 2: "POSTGRESQL", + 3: "DYNAMODB", + 4: "SQL_SERVER", + 5: "ORACLE", + } + MigrationData_Source_value = map[string]int32{ + "SOURCE_UNSPECIFIED": 0, + "MYSQL": 1, + "POSTGRESQL": 2, + "DYNAMODB": 3, + "SQL_SERVER": 4, + "ORACLE": 5, + } +) + +func (x MigrationData_Source) Enum() *MigrationData_Source { + p := new(MigrationData_Source) + *p = x + return p +} + +func (x MigrationData_Source) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MigrationData_Source) Descriptor() protoreflect.EnumDescriptor { + return file_migration_data_proto_enumTypes[0].Descriptor() +} + +func (MigrationData_Source) Type() protoreflect.EnumType { + return &file_migration_data_proto_enumTypes[0] +} + +func (x MigrationData_Source) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MigrationData_Source.Descriptor instead. +func (MigrationData_Source) EnumDescriptor() ([]byte, []int) { + return file_migration_data_proto_rawDescGZIP(), []int{0, 0} +} + +type MigrationData_SourceConnectionMechanism int32 + +const ( + MigrationData_SOURCE_CONNECTION_MECHANISM_UNSPECIFIED MigrationData_SourceConnectionMechanism = 0 + MigrationData_DIRECT_CONNECTION MigrationData_SourceConnectionMechanism = 1 + MigrationData_DB_DUMP MigrationData_SourceConnectionMechanism = 2 +) + +// Enum value maps for MigrationData_SourceConnectionMechanism. +var ( + MigrationData_SourceConnectionMechanism_name = map[int32]string{ + 0: "SOURCE_CONNECTION_MECHANISM_UNSPECIFIED", + 1: "DIRECT_CONNECTION", + 2: "DB_DUMP", + } + MigrationData_SourceConnectionMechanism_value = map[string]int32{ + "SOURCE_CONNECTION_MECHANISM_UNSPECIFIED": 0, + "DIRECT_CONNECTION": 1, + "DB_DUMP": 2, + } +) + +func (x MigrationData_SourceConnectionMechanism) Enum() *MigrationData_SourceConnectionMechanism { + p := new(MigrationData_SourceConnectionMechanism) + *p = x + return p +} + +func (x MigrationData_SourceConnectionMechanism) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MigrationData_SourceConnectionMechanism) Descriptor() protoreflect.EnumDescriptor { + return file_migration_data_proto_enumTypes[1].Descriptor() +} + +func (MigrationData_SourceConnectionMechanism) Type() protoreflect.EnumType { + return &file_migration_data_proto_enumTypes[1] +} + +func (x MigrationData_SourceConnectionMechanism) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MigrationData_SourceConnectionMechanism.Descriptor instead. +func (MigrationData_SourceConnectionMechanism) EnumDescriptor() ([]byte, []int) { + return file_migration_data_proto_rawDescGZIP(), []int{0, 1} +} + +type MigrationData_MigrationType int32 + +const ( + MigrationData_MIGRATION_TYPE_UNSPECIFIED MigrationData_MigrationType = 0 + MigrationData_SCHEMA_ONLY MigrationData_MigrationType = 1 + MigrationData_DATA_ONLY MigrationData_MigrationType = 2 + MigrationData_SCHEMA_AND_DATA MigrationData_MigrationType = 3 +) + +// Enum value maps for MigrationData_MigrationType. +var ( + MigrationData_MigrationType_name = map[int32]string{ + 0: "MIGRATION_TYPE_UNSPECIFIED", + 1: "SCHEMA_ONLY", + 2: "DATA_ONLY", + 3: "SCHEMA_AND_DATA", + } + MigrationData_MigrationType_value = map[string]int32{ + "MIGRATION_TYPE_UNSPECIFIED": 0, + "SCHEMA_ONLY": 1, + "DATA_ONLY": 2, + "SCHEMA_AND_DATA": 3, + } +) + +func (x MigrationData_MigrationType) Enum() *MigrationData_MigrationType { + p := new(MigrationData_MigrationType) + *p = x + return p +} + +func (x MigrationData_MigrationType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MigrationData_MigrationType) Descriptor() protoreflect.EnumDescriptor { + return file_migration_data_proto_enumTypes[2].Descriptor() +} + +func (MigrationData_MigrationType) Type() protoreflect.EnumType { + return &file_migration_data_proto_enumTypes[2] +} + +func (x MigrationData_MigrationType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MigrationData_MigrationType.Descriptor instead. +func (MigrationData_MigrationType) EnumDescriptor() ([]byte, []int) { + return file_migration_data_proto_rawDescGZIP(), []int{0, 2} +} + +type MigrationData_TargetDialect int32 + +const ( + MigrationData_TARGET_DIALECT_UNSPECIFIED MigrationData_TargetDialect = 0 + MigrationData_GOOGLE_STANDARD_SQL MigrationData_TargetDialect = 1 + MigrationData_POSTGRES MigrationData_TargetDialect = 2 +) + +// Enum value maps for MigrationData_TargetDialect. +var ( + MigrationData_TargetDialect_name = map[int32]string{ + 0: "TARGET_DIALECT_UNSPECIFIED", + 1: "GOOGLE_STANDARD_SQL", + 2: "POSTGRES", + } + MigrationData_TargetDialect_value = map[string]int32{ + "TARGET_DIALECT_UNSPECIFIED": 0, + "GOOGLE_STANDARD_SQL": 1, + "POSTGRES": 2, + } +) + +func (x MigrationData_TargetDialect) Enum() *MigrationData_TargetDialect { + p := new(MigrationData_TargetDialect) + *p = x + return p +} + +func (x MigrationData_TargetDialect) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MigrationData_TargetDialect) Descriptor() protoreflect.EnumDescriptor { + return file_migration_data_proto_enumTypes[3].Descriptor() +} + +func (MigrationData_TargetDialect) Type() protoreflect.EnumType { + return &file_migration_data_proto_enumTypes[3] +} + +func (x MigrationData_TargetDialect) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MigrationData_TargetDialect.Descriptor instead. +func (MigrationData_TargetDialect) EnumDescriptor() ([]byte, []int) { + return file_migration_data_proto_rawDescGZIP(), []int{0, 3} +} + +type MigrationData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // GUID with prefix ‘HB’ to identify that the request was made via HarbourBridge. + MigrationRequestId *string `protobuf:"bytes,1,opt,name=migration_request_id,json=migrationRequestId,proto3,oneof" json:"migration_request_id,omitempty"` + // Source for the migration. + Source *MigrationData_Source `protobuf:"varint,2,opt,name=source,proto3,enum=MigrationData_Source,oneof" json:"source,omitempty"` + // Mechanism used to fetch source data. + SourceConnectionMechanism *MigrationData_SourceConnectionMechanism `protobuf:"varint,3,opt,name=source_connection_mechanism,json=sourceConnectionMechanism,proto3,enum=MigrationData_SourceConnectionMechanism,oneof" json:"source_connection_mechanism,omitempty"` + // type of migration + MigrationType *MigrationData_MigrationType `protobuf:"varint,4,opt,name=migration_type,json=migrationType,proto3,enum=MigrationData_MigrationType,oneof" json:"migration_type,omitempty"` + // type of target dialect + TargetDialect *MigrationData_TargetDialect `protobuf:"varint,5,opt,name=target_dialect,json=targetDialect,proto3,enum=MigrationData_TargetDialect,oneof" json:"target_dialect,omitempty"` + SchemaPatterns *MigrationData_SchemaPatterns `protobuf:"bytes,6,opt,name=schema_patterns,json=schemaPatterns,proto3,oneof" json:"schema_patterns,omitempty"` + // size of each mutation - it will be populated from the writes_bytes we receive in the request + MutationSize *int64 `protobuf:"varint,7,opt,name=mutation_size,json=mutationSize,proto3,oneof" json:"mutation_size,omitempty"` +} + +func (x *MigrationData) Reset() { + *x = MigrationData{} + if protoimpl.UnsafeEnabled { + mi := &file_migration_data_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MigrationData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MigrationData) ProtoMessage() {} + +func (x *MigrationData) ProtoReflect() protoreflect.Message { + mi := &file_migration_data_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MigrationData.ProtoReflect.Descriptor instead. +func (*MigrationData) Descriptor() ([]byte, []int) { + return file_migration_data_proto_rawDescGZIP(), []int{0} +} + +func (x *MigrationData) GetMigrationRequestId() string { + if x != nil && x.MigrationRequestId != nil { + return *x.MigrationRequestId + } + return "" +} + +func (x *MigrationData) GetSource() MigrationData_Source { + if x != nil && x.Source != nil { + return *x.Source + } + return MigrationData_SOURCE_UNSPECIFIED +} + +func (x *MigrationData) GetSourceConnectionMechanism() MigrationData_SourceConnectionMechanism { + if x != nil && x.SourceConnectionMechanism != nil { + return *x.SourceConnectionMechanism + } + return MigrationData_SOURCE_CONNECTION_MECHANISM_UNSPECIFIED +} + +func (x *MigrationData) GetMigrationType() MigrationData_MigrationType { + if x != nil && x.MigrationType != nil { + return *x.MigrationType + } + return MigrationData_MIGRATION_TYPE_UNSPECIFIED +} + +func (x *MigrationData) GetTargetDialect() MigrationData_TargetDialect { + if x != nil && x.TargetDialect != nil { + return *x.TargetDialect + } + return MigrationData_TARGET_DIALECT_UNSPECIFIED +} + +func (x *MigrationData) GetSchemaPatterns() *MigrationData_SchemaPatterns { + if x != nil { + return x.SchemaPatterns + } + return nil +} + +func (x *MigrationData) GetMutationSize() int64 { + if x != nil && x.MutationSize != nil { + return *x.MutationSize + } + return 0 +} + +type MigrationData_SchemaPatterns struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // number of tables in the database being migrated + NumTables *int32 `protobuf:"varint,1,opt,name=num_tables,json=numTables,proto3,oneof" json:"num_tables,omitempty"` + MaxInterleaveDepth *int32 `protobuf:"varint,2,opt,name=max_interleave_depth,json=maxInterleaveDepth,proto3,oneof" json:"max_interleave_depth,omitempty"` + // number of interleaves in the database being migrated + NumInterleaves *int32 `protobuf:"varint,3,opt,name=num_interleaves,json=numInterleaves,proto3,oneof" json:"num_interleaves,omitempty"` + // number of indexes in the database being migrated + NumIndexes *int32 `protobuf:"varint,4,opt,name=num_indexes,json=numIndexes,proto3,oneof" json:"num_indexes,omitempty"` + // number of primary key constraints in the database being migrated + NumPrimaryKey *int32 `protobuf:"varint,5,opt,name=num_primary_key,json=numPrimaryKey,proto3,oneof" json:"num_primary_key,omitempty"` + // number of default constraints in the database being migrated + NumDefaultConstraint *int32 `protobuf:"varint,6,opt,name=num_default_constraint,json=numDefaultConstraint,proto3,oneof" json:"num_default_constraint,omitempty"` + // number of foreign key in the database being migrated + NumForeignKey *int32 `protobuf:"varint,7,opt,name=num_foreign_key,json=numForeignKey,proto3,oneof" json:"num_foreign_key,omitempty"` + // number of columns in the database + NumColumns *int32 `protobuf:"varint,8,opt,name=num_columns,json=numColumns,proto3,oneof" json:"num_columns,omitempty"` + // number of warnings occurred for the columns that didn’t map cleanly + NumWarnings *int32 `protobuf:"varint,9,opt,name=num_warnings,json=numWarnings,proto3,oneof" json:"num_warnings,omitempty"` + // indicates whether the source db schema had missing primary key + MissingPrimaryKey *bool `protobuf:"varint,10,opt,name=missing_primary_key,json=missingPrimaryKey,proto3,oneof" json:"missing_primary_key,omitempty"` +} + +func (x *MigrationData_SchemaPatterns) Reset() { + *x = MigrationData_SchemaPatterns{} + if protoimpl.UnsafeEnabled { + mi := &file_migration_data_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MigrationData_SchemaPatterns) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MigrationData_SchemaPatterns) ProtoMessage() {} + +func (x *MigrationData_SchemaPatterns) ProtoReflect() protoreflect.Message { + mi := &file_migration_data_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MigrationData_SchemaPatterns.ProtoReflect.Descriptor instead. +func (*MigrationData_SchemaPatterns) Descriptor() ([]byte, []int) { + return file_migration_data_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *MigrationData_SchemaPatterns) GetNumTables() int32 { + if x != nil && x.NumTables != nil { + return *x.NumTables + } + return 0 +} + +func (x *MigrationData_SchemaPatterns) GetMaxInterleaveDepth() int32 { + if x != nil && x.MaxInterleaveDepth != nil { + return *x.MaxInterleaveDepth + } + return 0 +} + +func (x *MigrationData_SchemaPatterns) GetNumInterleaves() int32 { + if x != nil && x.NumInterleaves != nil { + return *x.NumInterleaves + } + return 0 +} + +func (x *MigrationData_SchemaPatterns) GetNumIndexes() int32 { + if x != nil && x.NumIndexes != nil { + return *x.NumIndexes + } + return 0 +} + +func (x *MigrationData_SchemaPatterns) GetNumPrimaryKey() int32 { + if x != nil && x.NumPrimaryKey != nil { + return *x.NumPrimaryKey + } + return 0 +} + +func (x *MigrationData_SchemaPatterns) GetNumDefaultConstraint() int32 { + if x != nil && x.NumDefaultConstraint != nil { + return *x.NumDefaultConstraint + } + return 0 +} + +func (x *MigrationData_SchemaPatterns) GetNumForeignKey() int32 { + if x != nil && x.NumForeignKey != nil { + return *x.NumForeignKey + } + return 0 +} + +func (x *MigrationData_SchemaPatterns) GetNumColumns() int32 { + if x != nil && x.NumColumns != nil { + return *x.NumColumns + } + return 0 +} + +func (x *MigrationData_SchemaPatterns) GetNumWarnings() int32 { + if x != nil && x.NumWarnings != nil { + return *x.NumWarnings + } + return 0 +} + +func (x *MigrationData_SchemaPatterns) GetMissingPrimaryKey() bool { + if x != nil && x.MissingPrimaryKey != nil { + return *x.MissingPrimaryKey + } + return false +} + +var File_migration_data_proto protoreflect.FileDescriptor + +var file_migration_data_proto_rawDesc = []byte{ + 0x0a, 0x14, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb9, 0x0d, 0x0a, 0x0d, 0x4d, 0x69, 0x67, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x12, 0x35, 0x0a, 0x14, 0x6d, 0x69, 0x67, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x12, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, + 0x32, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x15, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x2e, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x01, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x88, 0x01, 0x01, 0x12, 0x6d, 0x0a, 0x1b, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, + 0x73, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x28, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, + 0x73, 0x6d, 0x48, 0x02, 0x52, 0x19, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x88, + 0x01, 0x01, 0x12, 0x48, 0x0a, 0x0e, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x4d, 0x69, 0x67, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x48, 0x03, 0x52, 0x0d, 0x6d, 0x69, 0x67, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x48, 0x0a, 0x0e, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x64, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x44, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x44, 0x69, 0x61, 0x6c, 0x65, + 0x63, 0x74, 0x48, 0x04, 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x44, 0x69, 0x61, 0x6c, + 0x65, 0x63, 0x74, 0x88, 0x01, 0x01, 0x12, 0x4b, 0x0a, 0x0f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x5f, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1d, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x2e, + 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x48, 0x05, + 0x52, 0x0e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, + 0x88, 0x01, 0x01, 0x12, 0x28, 0x0a, 0x0d, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x48, 0x06, 0x52, 0x0c, 0x6d, 0x75, + 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x69, 0x7a, 0x65, 0x88, 0x01, 0x01, 0x1a, 0x9f, 0x05, + 0x0a, 0x0e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, + 0x12, 0x22, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x73, 0x88, 0x01, 0x01, 0x12, 0x35, 0x0a, 0x14, 0x6d, 0x61, 0x78, 0x5f, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6c, 0x65, 0x61, 0x76, 0x65, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x05, 0x48, 0x01, 0x52, 0x12, 0x6d, 0x61, 0x78, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, + 0x61, 0x76, 0x65, 0x44, 0x65, 0x70, 0x74, 0x68, 0x88, 0x01, 0x01, 0x12, 0x2c, 0x0a, 0x0f, 0x6e, + 0x75, 0x6d, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, 0x61, 0x76, 0x65, 0x73, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x05, 0x48, 0x02, 0x52, 0x0e, 0x6e, 0x75, 0x6d, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x6c, 0x65, 0x61, 0x76, 0x65, 0x73, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, + 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x48, 0x03, + 0x52, 0x0a, 0x6e, 0x75, 0x6d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x88, 0x01, 0x01, 0x12, + 0x2b, 0x0a, 0x0f, 0x6e, 0x75, 0x6d, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x6b, + 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x48, 0x04, 0x52, 0x0d, 0x6e, 0x75, 0x6d, 0x50, + 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x4b, 0x65, 0x79, 0x88, 0x01, 0x01, 0x12, 0x39, 0x0a, 0x16, + 0x6e, 0x75, 0x6d, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x48, 0x05, 0x52, 0x14, + 0x6e, 0x75, 0x6d, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, + 0x61, 0x69, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, 0x2b, 0x0a, 0x0f, 0x6e, 0x75, 0x6d, 0x5f, 0x66, + 0x6f, 0x72, 0x65, 0x69, 0x67, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, + 0x48, 0x06, 0x52, 0x0d, 0x6e, 0x75, 0x6d, 0x46, 0x6f, 0x72, 0x65, 0x69, 0x67, 0x6e, 0x4b, 0x65, + 0x79, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x6f, 0x6c, 0x75, + 0x6d, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x48, 0x07, 0x52, 0x0a, 0x6e, 0x75, 0x6d, + 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x88, 0x01, 0x01, 0x12, 0x26, 0x0a, 0x0c, 0x6e, 0x75, + 0x6d, 0x5f, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, + 0x48, 0x08, 0x52, 0x0b, 0x6e, 0x75, 0x6d, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x88, + 0x01, 0x01, 0x12, 0x33, 0x0a, 0x13, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x72, + 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x48, + 0x09, 0x52, 0x11, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, + 0x79, 0x4b, 0x65, 0x79, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, + 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, 0x61, 0x76, 0x65, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x42, + 0x12, 0x0a, 0x10, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, 0x61, + 0x76, 0x65, 0x73, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x69, 0x6e, 0x64, 0x65, + 0x78, 0x65, 0x73, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x70, 0x72, 0x69, 0x6d, + 0x61, 0x72, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x42, 0x19, 0x0a, 0x17, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, + 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, + 0x6e, 0x74, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x66, 0x6f, 0x72, 0x65, 0x69, + 0x67, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x63, + 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x77, + 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x6d, 0x69, 0x73, 0x73, + 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x22, + 0x65, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x59, 0x53, 0x51, 0x4c, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, + 0x50, 0x4f, 0x53, 0x54, 0x47, 0x52, 0x45, 0x53, 0x51, 0x4c, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, + 0x44, 0x59, 0x4e, 0x41, 0x4d, 0x4f, 0x44, 0x42, 0x10, 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x51, + 0x4c, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x52, + 0x41, 0x43, 0x4c, 0x45, 0x10, 0x05, 0x22, 0x6c, 0x0a, 0x19, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x63, 0x68, 0x61, 0x6e, + 0x69, 0x73, 0x6d, 0x12, 0x2b, 0x0a, 0x27, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x43, 0x4f, + 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x43, 0x48, 0x41, 0x4e, 0x49, + 0x53, 0x4d, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, + 0x12, 0x15, 0x0a, 0x11, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, + 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x42, 0x5f, 0x44, 0x55, + 0x4d, 0x50, 0x10, 0x02, 0x22, 0x64, 0x0a, 0x0d, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x1a, 0x4d, 0x49, 0x47, 0x52, 0x41, 0x54, 0x49, + 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, + 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, + 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x4f, + 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, + 0x41, 0x4e, 0x44, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x10, 0x03, 0x22, 0x56, 0x0a, 0x0d, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x12, 0x1e, 0x0a, 0x1a, 0x54, + 0x41, 0x52, 0x47, 0x45, 0x54, 0x5f, 0x44, 0x49, 0x41, 0x4c, 0x45, 0x43, 0x54, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x47, + 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x4e, 0x44, 0x41, 0x52, 0x44, 0x5f, 0x53, + 0x51, 0x4c, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x4f, 0x53, 0x54, 0x47, 0x52, 0x45, 0x53, + 0x10, 0x02, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x42, 0x09, 0x0a, 0x07, 0x5f, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x1e, 0x0a, 0x1c, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, 0x63, + 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x6d, 0x69, 0x67, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x5f, 0x64, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x42, 0x12, 0x0a, 0x10, + 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, + 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x69, + 0x7a, 0x65, 0x42, 0x0c, 0x5a, 0x0a, 0x2f, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_migration_data_proto_rawDescOnce sync.Once + file_migration_data_proto_rawDescData = file_migration_data_proto_rawDesc +) + +func file_migration_data_proto_rawDescGZIP() []byte { + file_migration_data_proto_rawDescOnce.Do(func() { + file_migration_data_proto_rawDescData = protoimpl.X.CompressGZIP(file_migration_data_proto_rawDescData) + }) + return file_migration_data_proto_rawDescData +} + +var file_migration_data_proto_enumTypes = make([]protoimpl.EnumInfo, 4) +var file_migration_data_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_migration_data_proto_goTypes = []interface{}{ + (MigrationData_Source)(0), // 0: MigrationData.Source + (MigrationData_SourceConnectionMechanism)(0), // 1: MigrationData.SourceConnectionMechanism + (MigrationData_MigrationType)(0), // 2: MigrationData.MigrationType + (MigrationData_TargetDialect)(0), // 3: MigrationData.TargetDialect + (*MigrationData)(nil), // 4: MigrationData + (*MigrationData_SchemaPatterns)(nil), // 5: MigrationData.SchemaPatterns +} +var file_migration_data_proto_depIdxs = []int32{ + 0, // 0: MigrationData.source:type_name -> MigrationData.Source + 1, // 1: MigrationData.source_connection_mechanism:type_name -> MigrationData.SourceConnectionMechanism + 2, // 2: MigrationData.migration_type:type_name -> MigrationData.MigrationType + 3, // 3: MigrationData.target_dialect:type_name -> MigrationData.TargetDialect + 5, // 4: MigrationData.schema_patterns:type_name -> MigrationData.SchemaPatterns + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_migration_data_proto_init() } +func file_migration_data_proto_init() { + if File_migration_data_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_migration_data_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MigrationData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_migration_data_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MigrationData_SchemaPatterns); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_migration_data_proto_msgTypes[0].OneofWrappers = []interface{}{} + file_migration_data_proto_msgTypes[1].OneofWrappers = []interface{}{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_migration_data_proto_rawDesc, + NumEnums: 4, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_migration_data_proto_goTypes, + DependencyIndexes: file_migration_data_proto_depIdxs, + EnumInfos: file_migration_data_proto_enumTypes, + MessageInfos: file_migration_data_proto_msgTypes, + }.Build() + File_migration_data_proto = out.File + file_migration_data_proto_rawDesc = nil + file_migration_data_proto_goTypes = nil + file_migration_data_proto_depIdxs = nil +} diff --git a/proto/migration_data.proto b/proto/migration_data.proto new file mode 100644 index 000000000..c68af75dd --- /dev/null +++ b/proto/migration_data.proto @@ -0,0 +1,71 @@ +syntax = "proto3"; + +option go_package = "/migration"; + +message MigrationData { + + // GUID with prefix ‘HB’ to identify that the request was made via HarbourBridge. + optional string migration_request_id = 1; + + enum Source { + SOURCE_UNSPECIFIED = 0; + MYSQL = 1; + POSTGRESQL = 2; + DYNAMODB = 3; + SQL_SERVER = 4; + ORACLE = 5; + } + // Source for the migration. + optional Source source = 2; + + enum SourceConnectionMechanism { + SOURCE_CONNECTION_MECHANISM_UNSPECIFIED = 0; + DIRECT_CONNECTION = 1; + DB_DUMP = 2; + } + // Mechanism used to fetch source data. + optional SourceConnectionMechanism source_connection_mechanism = 3; + + enum MigrationType { + MIGRATION_TYPE_UNSPECIFIED = 0; + SCHEMA_ONLY = 1; + DATA_ONLY = 2; + SCHEMA_AND_DATA = 3; + } + // type of migration + optional MigrationType migration_type = 4; + + enum TargetDialect { + TARGET_DIALECT_UNSPECIFIED = 0; + GOOGLE_STANDARD_SQL = 1; + POSTGRES = 2; + } + // type of target dialect + optional TargetDialect target_dialect = 5; + + message SchemaPatterns { + // number of tables in the database being migrated + optional int32 num_tables = 1; + optional int32 max_interleave_depth = 2; + // number of interleaves in the database being migrated + optional int32 num_interleaves = 3; + // number of indexes in the database being migrated + optional int32 num_indexes = 4; + // number of primary key constraints in the database being migrated + optional int32 num_primary_key = 5; + // number of default constraints in the database being migrated + optional int32 num_default_constraint = 6; + // number of foreign key in the database being migrated + optional int32 num_foreign_key = 7; + // number of columns in the database + optional int32 num_columns = 8; + // number of warnings occurred for the columns that didn’t map cleanly + optional int32 num_warnings = 9; + // indicates whether the source db schema had missing primary key + optional bool missing_primary_key = 10; + } + optional SchemaPatterns schema_patterns = 6; + + // size of each mutation - it will be populated from the writes_bytes we receive in the request + optional int64 mutation_size = 7; + } From 981eedebf4faee14991d158be5848e88ab067d9e Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Wed, 2 Mar 2022 15:31:24 +0530 Subject: [PATCH 09/20] Adding metrics to HarbourBridge --- cmd/data.go | 6 + cmd/schema_and_data.go | 5 + common/utils/utils.go | 103 ++++++++++++- conversion/conversion.go | 25 +++- go.mod | 1 + internal/convert.go | 12 +- internal/report.go | 5 + proto/migration/migration_data.pb.go | 215 +++++++++++++-------------- proto/migration_data.proto | 39 ++--- 9 files changed, 272 insertions(+), 139 deletions(-) diff --git a/cmd/data.go b/cmd/data.go index a0118fc0d..d06e12bef 100644 --- a/cmd/data.go +++ b/cmd/data.go @@ -13,6 +13,7 @@ import ( "github.com/cloudspannerecosystem/harbourbridge/conversion" "github.com/cloudspannerecosystem/harbourbridge/internal" "github.com/cloudspannerecosystem/harbourbridge/profiles" + "github.com/cloudspannerecosystem/harbourbridge/proto/migration" "github.com/google/subcommands" ) @@ -137,6 +138,11 @@ func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface } defer adminClient.Close() + // Populating migration dara in conv. + utils.PopulateMigrationData(conv, sourceProfile.Driver, targetProfile.TargetDb) + conv.MigrationData.MigrationType = migration.MigrationData_DATA_ONLY.Enum() + conversion.Report(sourceProfile.Driver, nil, ioHelper.BytesRead, "", conv, cmd.filePrefix+reportFile, ioHelper.Out) + if !sourceProfile.UseTargetSchema() { err = conversion.CreateOrUpdateDatabase(ctx, adminClient, dbURI, conv, ioHelper.Out) if err != nil { diff --git a/cmd/schema_and_data.go b/cmd/schema_and_data.go index eb27c532c..4426a9e62 100644 --- a/cmd/schema_and_data.go +++ b/cmd/schema_and_data.go @@ -13,6 +13,7 @@ import ( "github.com/cloudspannerecosystem/harbourbridge/conversion" "github.com/cloudspannerecosystem/harbourbridge/internal" "github.com/cloudspannerecosystem/harbourbridge/profiles" + "github.com/cloudspannerecosystem/harbourbridge/proto/migration" "github.com/google/subcommands" ) @@ -110,6 +111,10 @@ func (cmd *SchemaAndDataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ... panic(err) } + // Populating migration dara in conv. + utils.PopulateMigrationData(conv, sourceProfile.Driver, targetProfile.TargetDb) + conv.MigrationData.MigrationType = migration.MigrationData_SCHEMA_AND_DATA.Enum() + conversion.WriteSchemaFile(conv, schemaConversionStartTime, cmd.filePrefix+schemaFile, ioHelper.Out) conversion.WriteSessionFile(conv, cmd.filePrefix+sessionFile, ioHelper.Out) conversion.Report(sourceProfile.Driver, nil, ioHelper.BytesRead, "", conv, cmd.filePrefix+reportFile, ioHelper.Out) diff --git a/common/utils/utils.go b/common/utils/utils.go index 0307acb35..4cb4c9758 100644 --- a/common/utils/utils.go +++ b/common/utils/utils.go @@ -5,11 +5,12 @@ package utils import ( "bufio" "context" - "crypto/rand" "fmt" "io" "io/ioutil" "log" + "math" + "math/rand" "net/url" "os" "os/exec" @@ -25,6 +26,7 @@ import ( "cloud.google.com/go/storage" "github.com/cloudspannerecosystem/harbourbridge/common/constants" "github.com/cloudspannerecosystem/harbourbridge/internal" + "github.com/cloudspannerecosystem/harbourbridge/proto/migration" "github.com/cloudspannerecosystem/harbourbridge/sources/common" "github.com/cloudspannerecosystem/harbourbridge/sources/spanner" "golang.org/x/crypto/ssh/terminal" @@ -492,3 +494,102 @@ func DialectToTarget(dialect string) string { } return constants.TargetSpanner } + +const alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890" + +func randomString(n int) string { + var sb strings.Builder + k := len(alphabet) + + for i := 0; i < n; i++ { + c := alphabet[rand.Intn(k)] + sb.WriteByte(c) + } + + return sb.String() +} + +func PopulateMigrationData(conv *internal.Conv, driver, targetDb string) { + + migrationRequestId := "HB" + randomString(14) + + numTables := int32(len(conv.SrcSchema)) + migrationData := migration.MigrationData{ + MigrationRequestId: &migrationRequestId, + SchemaPatterns: &migration.MigrationData_SchemaPatterns{ + NumTables: &numTables, + }, + } + switch driver { + case constants.PGDUMP: + migrationData.SourceConnectionMechanism = migration.MigrationData_DB_DUMP.Enum() + migrationData.Source = migration.MigrationData_POSTGRESQL.Enum() + case constants.MYSQLDUMP: + migrationData.SourceConnectionMechanism = migration.MigrationData_DB_DUMP.Enum() + migrationData.Source = migration.MigrationData_MYSQL.Enum() + case constants.POSTGRES: + migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() + migrationData.Source = migration.MigrationData_POSTGRESQL.Enum() + case constants.MYSQL: + migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() + migrationData.Source = migration.MigrationData_MYSQL.Enum() + case constants.DYNAMODB: + migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() + migrationData.Source = migration.MigrationData_DYNAMODB.Enum() + case constants.ORACLE: + migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() + migrationData.Source = migration.MigrationData_ORACLE.Enum() + case constants.SQLSERVER: + migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() + migrationData.Source = migration.MigrationData_SQL_SERVER.Enum() + case constants.CSV: + migrationData.SourceConnectionMechanism = migration.MigrationData_FILE.Enum() + migrationData.Source = migration.MigrationData_CSV.Enum() + } + + switch targetDb { + case constants.TargetSpanner: + migrationData.TargetDialect = migration.MigrationData_GOOGLE_STANDARD_SQL.Enum() + case constants.TargetExperimentalPostgres: + migrationData.TargetDialect = migration.MigrationData_POSTGRES.Enum() + } + + var numForeignKey, numIndexes, numPrimaryKey, numInterleaves, maxInterleaveDepth int32 = 0, 0, 0, 0, 0 + missingPrimaryKey := false + + for _, table := range conv.SrcSchema { + if len(table.ForeignKeys) != 0 { + numForeignKey++ + } + if len(table.PrimaryKeys) != 0 { + numPrimaryKey++ + } + numIndexes += int32(len(table.Indexes)) + } + + for _, table := range conv.SpSchema { + if table.Parent != "" { + numInterleaves++ + } + } + + for _, table := range conv.SpSchema { + if table.Parent != "" { + depth := 1 + parentTableName := table.Parent + for conv.SpSchema[parentTableName].Parent != "" { + depth++ + parentTableName = conv.SpSchema[parentTableName].Parent + } + maxInterleaveDepth = int32(math.Max(float64(maxInterleaveDepth), float64(depth))) + } + } + + migrationData.SchemaPatterns.NumForeignKey = &numForeignKey + migrationData.SchemaPatterns.NumPrimaryKey = &numPrimaryKey + migrationData.SchemaPatterns.NumIndexes = &numIndexes + migrationData.SchemaPatterns.NumInterleaves = &numInterleaves + migrationData.SchemaPatterns.MissingPrimaryKey = &missingPrimaryKey + migrationData.SchemaPatterns.MaxInterleaveDepth = &maxInterleaveDepth + conv.MigrationData = migrationData +} diff --git a/conversion/conversion.go b/conversion/conversion.go index d7981751f..0cc2d3234 100644 --- a/conversion/conversion.go +++ b/conversion/conversion.go @@ -43,11 +43,13 @@ import ( "github.com/aws/aws-sdk-go/aws/session" dydb "github.com/aws/aws-sdk-go/service/dynamodb" adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" + "google.golang.org/grpc/metadata" "github.com/cloudspannerecosystem/harbourbridge/common/constants" "github.com/cloudspannerecosystem/harbourbridge/common/utils" "github.com/cloudspannerecosystem/harbourbridge/internal" "github.com/cloudspannerecosystem/harbourbridge/profiles" + "github.com/cloudspannerecosystem/harbourbridge/proto/migration" "github.com/cloudspannerecosystem/harbourbridge/sources/common" "github.com/cloudspannerecosystem/harbourbridge/sources/csv" "github.com/cloudspannerecosystem/harbourbridge/sources/dynamodb" @@ -67,6 +69,8 @@ var ( MaxWorkers = 20 ) +const migrationMetadataKey = "cloud-spanner-migration-metadata" + // SchemaConv performs the schema conversion // TODO: Pass around cmd.SourceProfile instead of sqlConnectionStr and schemaSampleSize. // Doing that requires refactoring since that would introduce a circular dependency between @@ -188,7 +192,11 @@ func dataFromDatabase(sourceProfile profiles.SourceProfile, config writer.BatchW p := internal.NewProgress(totalRows, "Writing data to Spanner", internal.Verbose(), false) rows := int64(0) config.Write = func(m []*sp.Mutation) error { - _, err := client.Apply(context.Background(), m) + migrationData := migration.MigrationData{ + MigrationRequestId: conv.MigrationData.MigrationRequestId, + } + migrationMetadataValue := migrationData.String() + _, err := client.Apply(metadata.AppendToOutgoingContext(context.Background(), migrationMetadataKey, migrationMetadataValue), m) if err != nil { return err } @@ -268,7 +276,11 @@ func dataFromDump(driver string, config writer.BatchWriterConfig, ioHelper *util r := internal.NewReader(bufio.NewReader(ioHelper.SeekableIn), nil) rows := int64(0) config.Write = func(m []*sp.Mutation) error { - _, err := client.Apply(context.Background(), m) + migrationData := migration.MigrationData{ + MigrationRequestId: conv.MigrationData.MigrationRequestId, + } + migrationMetadataValue := migrationData.String() + _, err := client.Apply(metadata.AppendToOutgoingContext(context.Background(), migrationMetadataKey, migrationMetadataValue), m) if err != nil { return err } @@ -332,7 +344,11 @@ func dataFromCSV(ctx context.Context, sourceProfile profiles.SourceProfile, targ p := internal.NewProgress(totalRows, "Writing data to Spanner", internal.Verbose(), false) rows := int64(0) config.Write = func(m []*sp.Mutation) error { - _, err := client.Apply(context.Background(), m) + migrationData := migration.MigrationData{ + MigrationRequestId: conv.MigrationData.MigrationRequestId, + } + migrationMetadataValue := migrationData.String() + _, err := client.Apply(metadata.AppendToOutgoingContext(context.Background(), migrationMetadataKey, migrationMetadataValue), m) if err != nil { return err } @@ -466,6 +482,9 @@ func CreateOrUpdateDatabase(ctx context.Context, adminClient *database.DatabaseA if err != nil { return err } + // Adding migration metadata to the outgoing context. + migrationMetadataValue := conv.MigrationData.String() + ctx = metadata.AppendToOutgoingContext(ctx, migrationMetadataKey, migrationMetadataValue) if dbExists { err := UpdateDatabase(ctx, adminClient, dbURI, conv, out) if err != nil { diff --git a/go.mod b/go.mod index 0a60442e0..f4ada8725 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( golang.org/x/tools v0.1.7 // indirect google.golang.org/api v0.57.0 google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 + google.golang.org/grpc v1.40.0 // indirect google.golang.org/protobuf v1.27.1 // indirect honnef.co/go/tools v0.2.1 // indirect ) diff --git a/internal/convert.go b/internal/convert.go index f2f62c9bc..3b185d99f 100644 --- a/internal/convert.go +++ b/internal/convert.go @@ -18,6 +18,7 @@ import ( "fmt" "time" + "github.com/cloudspannerecosystem/harbourbridge/proto/migration" "github.com/cloudspannerecosystem/harbourbridge/schema" "github.com/cloudspannerecosystem/harbourbridge/spanner/ddl" ) @@ -37,11 +38,12 @@ type Conv struct { Location *time.Location // Timezone (for timestamp conversion). sampleBadRows rowSamples // Rows that generated errors during conversion. Stats stats - TimezoneOffset string // Timezone offset for timestamp conversion. - TargetDb string // The target database to which HarbourBridge is writing. - UniquePKey map[string][]string // Maps Spanner table name to unique column name being used as primary key (if needed). - SchemaConversionDuration time.Duration `json:"-"` // Duration of schema conversion. - DataConversionDuration time.Duration `json:"-"` // Duration of data conversion. + TimezoneOffset string // Timezone offset for timestamp conversion. + TargetDb string // The target database to which HarbourBridge is writing. + UniquePKey map[string][]string // Maps Spanner table name to unique column name being used as primary key (if needed). + SchemaConversionDuration time.Duration `json:"-"` // Duration of schema conversion. + DataConversionDuration time.Duration `json:"-"` // Duration of data conversion. + MigrationData migration.MigrationData `json:"-"` // Migration related metadata to be sent to spanner. } type mode int diff --git a/internal/report.go b/internal/report.go index f154b877e..3b642ab37 100644 --- a/internal/report.go +++ b/internal/report.go @@ -396,6 +396,7 @@ func GenerateSummary(conv *Conv, r []tableReport, badWrites map[string]int64) st warnings += t.Warnings * weight if t.SyntheticPKey != "" { missingPKey = true + conv.MigrationData.SchemaPatterns.MissingPrimaryKey = &missingPKey } } // Don't use tableReport for rows/badRows stats because tableReport @@ -408,6 +409,10 @@ func GenerateSummary(conv *Conv, r []tableReport, badWrites map[string]int64) st for _, n := range badWrites { badRows += n } + numColumns := int32(int(cols)) + numWarnings := int32(int(warnings)) + conv.MigrationData.SchemaPatterns.NumColumns = &numColumns + conv.MigrationData.SchemaPatterns.NumWarnings = &numWarnings return rateConversion(rows, badRows, cols, warnings, missingPKey, true, conv.SchemaMode()) } diff --git a/proto/migration/migration_data.pb.go b/proto/migration/migration_data.pb.go index 051e02dc9..c2ee5b8ab 100644 --- a/proto/migration/migration_data.pb.go +++ b/proto/migration/migration_data.pb.go @@ -29,6 +29,7 @@ const ( MigrationData_DYNAMODB MigrationData_Source = 3 MigrationData_SQL_SERVER MigrationData_Source = 4 MigrationData_ORACLE MigrationData_Source = 5 + MigrationData_CSV MigrationData_Source = 6 ) // Enum value maps for MigrationData_Source. @@ -40,6 +41,7 @@ var ( 3: "DYNAMODB", 4: "SQL_SERVER", 5: "ORACLE", + 6: "CSV", } MigrationData_Source_value = map[string]int32{ "SOURCE_UNSPECIFIED": 0, @@ -48,6 +50,7 @@ var ( "DYNAMODB": 3, "SQL_SERVER": 4, "ORACLE": 5, + "CSV": 6, } ) @@ -84,6 +87,7 @@ const ( MigrationData_SOURCE_CONNECTION_MECHANISM_UNSPECIFIED MigrationData_SourceConnectionMechanism = 0 MigrationData_DIRECT_CONNECTION MigrationData_SourceConnectionMechanism = 1 MigrationData_DB_DUMP MigrationData_SourceConnectionMechanism = 2 + MigrationData_FILE MigrationData_SourceConnectionMechanism = 3 ) // Enum value maps for MigrationData_SourceConnectionMechanism. @@ -92,11 +96,13 @@ var ( 0: "SOURCE_CONNECTION_MECHANISM_UNSPECIFIED", 1: "DIRECT_CONNECTION", 2: "DB_DUMP", + 3: "FILE", } MigrationData_SourceConnectionMechanism_value = map[string]int32{ "SOURCE_CONNECTION_MECHANISM_UNSPECIFIED": 0, "DIRECT_CONNECTION": 1, "DB_DUMP": 2, + "FILE": 3, } ) @@ -233,19 +239,20 @@ type MigrationData struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // GUID with prefix ‘HB’ to identify that the request was made via HarbourBridge. + // A system generated id with prefix ‘HB’ to identify that the request was + // made via HarbourBridge. MigrationRequestId *string `protobuf:"bytes,1,opt,name=migration_request_id,json=migrationRequestId,proto3,oneof" json:"migration_request_id,omitempty"` // Source for the migration. Source *MigrationData_Source `protobuf:"varint,2,opt,name=source,proto3,enum=MigrationData_Source,oneof" json:"source,omitempty"` // Mechanism used to fetch source data. SourceConnectionMechanism *MigrationData_SourceConnectionMechanism `protobuf:"varint,3,opt,name=source_connection_mechanism,json=sourceConnectionMechanism,proto3,enum=MigrationData_SourceConnectionMechanism,oneof" json:"source_connection_mechanism,omitempty"` - // type of migration + // Type of migration. MigrationType *MigrationData_MigrationType `protobuf:"varint,4,opt,name=migration_type,json=migrationType,proto3,enum=MigrationData_MigrationType,oneof" json:"migration_type,omitempty"` - // type of target dialect + // Type of target dialect. TargetDialect *MigrationData_TargetDialect `protobuf:"varint,5,opt,name=target_dialect,json=targetDialect,proto3,enum=MigrationData_TargetDialect,oneof" json:"target_dialect,omitempty"` SchemaPatterns *MigrationData_SchemaPatterns `protobuf:"bytes,6,opt,name=schema_patterns,json=schemaPatterns,proto3,oneof" json:"schema_patterns,omitempty"` - // size of each mutation - it will be populated from the writes_bytes we receive in the request - MutationSize *int64 `protobuf:"varint,7,opt,name=mutation_size,json=mutationSize,proto3,oneof" json:"mutation_size,omitempty"` + // Size of each commit in bytes. + WriteBytes *int64 `protobuf:"varint,7,opt,name=write_bytes,json=writeBytes,proto3,oneof" json:"write_bytes,omitempty"` } func (x *MigrationData) Reset() { @@ -322,9 +329,9 @@ func (x *MigrationData) GetSchemaPatterns() *MigrationData_SchemaPatterns { return nil } -func (x *MigrationData) GetMutationSize() int64 { - if x != nil && x.MutationSize != nil { - return *x.MutationSize +func (x *MigrationData) GetWriteBytes() int64 { + if x != nil && x.WriteBytes != nil { + return *x.WriteBytes } return 0 } @@ -334,25 +341,23 @@ type MigrationData_SchemaPatterns struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // number of tables in the database being migrated + // Number of tables in the database being migrated. NumTables *int32 `protobuf:"varint,1,opt,name=num_tables,json=numTables,proto3,oneof" json:"num_tables,omitempty"` MaxInterleaveDepth *int32 `protobuf:"varint,2,opt,name=max_interleave_depth,json=maxInterleaveDepth,proto3,oneof" json:"max_interleave_depth,omitempty"` - // number of interleaves in the database being migrated + // Number of interleaves in the database being migrated. NumInterleaves *int32 `protobuf:"varint,3,opt,name=num_interleaves,json=numInterleaves,proto3,oneof" json:"num_interleaves,omitempty"` - // number of indexes in the database being migrated + // Number of indexes in the database being migrated. NumIndexes *int32 `protobuf:"varint,4,opt,name=num_indexes,json=numIndexes,proto3,oneof" json:"num_indexes,omitempty"` - // number of primary key constraints in the database being migrated + // Number of primary key constraints in the database being migrated. NumPrimaryKey *int32 `protobuf:"varint,5,opt,name=num_primary_key,json=numPrimaryKey,proto3,oneof" json:"num_primary_key,omitempty"` - // number of default constraints in the database being migrated - NumDefaultConstraint *int32 `protobuf:"varint,6,opt,name=num_default_constraint,json=numDefaultConstraint,proto3,oneof" json:"num_default_constraint,omitempty"` - // number of foreign key in the database being migrated - NumForeignKey *int32 `protobuf:"varint,7,opt,name=num_foreign_key,json=numForeignKey,proto3,oneof" json:"num_foreign_key,omitempty"` - // number of columns in the database - NumColumns *int32 `protobuf:"varint,8,opt,name=num_columns,json=numColumns,proto3,oneof" json:"num_columns,omitempty"` - // number of warnings occurred for the columns that didn’t map cleanly - NumWarnings *int32 `protobuf:"varint,9,opt,name=num_warnings,json=numWarnings,proto3,oneof" json:"num_warnings,omitempty"` - // indicates whether the source db schema had missing primary key - MissingPrimaryKey *bool `protobuf:"varint,10,opt,name=missing_primary_key,json=missingPrimaryKey,proto3,oneof" json:"missing_primary_key,omitempty"` + // Number of foreign key in the database being migrated. + NumForeignKey *int32 `protobuf:"varint,6,opt,name=num_foreign_key,json=numForeignKey,proto3,oneof" json:"num_foreign_key,omitempty"` + // Number of columns in the database. + NumColumns *int32 `protobuf:"varint,7,opt,name=num_columns,json=numColumns,proto3,oneof" json:"num_columns,omitempty"` + // Number of warnings occurred for the columns that didn’t map cleanly. + NumWarnings *int32 `protobuf:"varint,8,opt,name=num_warnings,json=numWarnings,proto3,oneof" json:"num_warnings,omitempty"` + // Indicates whether the source db schema had missing primary key. + MissingPrimaryKey *bool `protobuf:"varint,9,opt,name=missing_primary_key,json=missingPrimaryKey,proto3,oneof" json:"missing_primary_key,omitempty"` } func (x *MigrationData_SchemaPatterns) Reset() { @@ -422,13 +427,6 @@ func (x *MigrationData_SchemaPatterns) GetNumPrimaryKey() int32 { return 0 } -func (x *MigrationData_SchemaPatterns) GetNumDefaultConstraint() int32 { - if x != nil && x.NumDefaultConstraint != nil { - return *x.NumDefaultConstraint - } - return 0 -} - func (x *MigrationData_SchemaPatterns) GetNumForeignKey() int32 { if x != nil && x.NumForeignKey != nil { return *x.NumForeignKey @@ -461,7 +459,7 @@ var File_migration_data_proto protoreflect.FileDescriptor var file_migration_data_proto_rawDesc = []byte{ 0x0a, 0x14, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb9, 0x0d, 0x0a, 0x0d, 0x4d, 0x69, 0x67, 0x72, 0x61, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xf0, 0x0c, 0x0a, 0x0d, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x12, 0x35, 0x0a, 0x14, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x12, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, @@ -490,87 +488,82 @@ var file_migration_data_proto_rawDesc = []byte{ 0x1d, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x48, 0x05, 0x52, 0x0e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, - 0x88, 0x01, 0x01, 0x12, 0x28, 0x0a, 0x0d, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x73, 0x69, 0x7a, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x48, 0x06, 0x52, 0x0c, 0x6d, 0x75, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x69, 0x7a, 0x65, 0x88, 0x01, 0x01, 0x1a, 0x9f, 0x05, - 0x0a, 0x0e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, - 0x12, 0x22, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x54, 0x61, 0x62, 0x6c, 0x65, - 0x73, 0x88, 0x01, 0x01, 0x12, 0x35, 0x0a, 0x14, 0x6d, 0x61, 0x78, 0x5f, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6c, 0x65, 0x61, 0x76, 0x65, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x05, 0x48, 0x01, 0x52, 0x12, 0x6d, 0x61, 0x78, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, - 0x61, 0x76, 0x65, 0x44, 0x65, 0x70, 0x74, 0x68, 0x88, 0x01, 0x01, 0x12, 0x2c, 0x0a, 0x0f, 0x6e, - 0x75, 0x6d, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, 0x61, 0x76, 0x65, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x05, 0x48, 0x02, 0x52, 0x0e, 0x6e, 0x75, 0x6d, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x6c, 0x65, 0x61, 0x76, 0x65, 0x73, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, - 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x48, 0x03, - 0x52, 0x0a, 0x6e, 0x75, 0x6d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x88, 0x01, 0x01, 0x12, - 0x2b, 0x0a, 0x0f, 0x6e, 0x75, 0x6d, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x6b, - 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x48, 0x04, 0x52, 0x0d, 0x6e, 0x75, 0x6d, 0x50, - 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x4b, 0x65, 0x79, 0x88, 0x01, 0x01, 0x12, 0x39, 0x0a, 0x16, - 0x6e, 0x75, 0x6d, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x48, 0x05, 0x52, 0x14, - 0x6e, 0x75, 0x6d, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, - 0x61, 0x69, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, 0x2b, 0x0a, 0x0f, 0x6e, 0x75, 0x6d, 0x5f, 0x66, - 0x6f, 0x72, 0x65, 0x69, 0x67, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, - 0x48, 0x06, 0x52, 0x0d, 0x6e, 0x75, 0x6d, 0x46, 0x6f, 0x72, 0x65, 0x69, 0x67, 0x6e, 0x4b, 0x65, - 0x79, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x6f, 0x6c, 0x75, - 0x6d, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x48, 0x07, 0x52, 0x0a, 0x6e, 0x75, 0x6d, - 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x88, 0x01, 0x01, 0x12, 0x26, 0x0a, 0x0c, 0x6e, 0x75, - 0x6d, 0x5f, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, - 0x48, 0x08, 0x52, 0x0b, 0x6e, 0x75, 0x6d, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x88, - 0x01, 0x01, 0x12, 0x33, 0x0a, 0x13, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x72, - 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x48, - 0x09, 0x52, 0x11, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, - 0x79, 0x4b, 0x65, 0x79, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, 0x61, 0x76, 0x65, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x42, - 0x12, 0x0a, 0x10, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, 0x61, - 0x76, 0x65, 0x73, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x69, 0x6e, 0x64, 0x65, - 0x78, 0x65, 0x73, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x70, 0x72, 0x69, 0x6d, - 0x61, 0x72, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x42, 0x19, 0x0a, 0x17, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, - 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, - 0x6e, 0x74, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x66, 0x6f, 0x72, 0x65, 0x69, - 0x67, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x63, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x77, - 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x6d, 0x69, 0x73, 0x73, - 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x22, - 0x65, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, - 0x00, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x59, 0x53, 0x51, 0x4c, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, - 0x50, 0x4f, 0x53, 0x54, 0x47, 0x52, 0x45, 0x53, 0x51, 0x4c, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, - 0x44, 0x59, 0x4e, 0x41, 0x4d, 0x4f, 0x44, 0x42, 0x10, 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x51, - 0x4c, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x52, - 0x41, 0x43, 0x4c, 0x45, 0x10, 0x05, 0x22, 0x6c, 0x0a, 0x19, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x63, 0x68, 0x61, 0x6e, - 0x69, 0x73, 0x6d, 0x12, 0x2b, 0x0a, 0x27, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x43, 0x4f, - 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x43, 0x48, 0x41, 0x4e, 0x49, - 0x53, 0x4d, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x15, 0x0a, 0x11, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, - 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x42, 0x5f, 0x44, 0x55, - 0x4d, 0x50, 0x10, 0x02, 0x22, 0x64, 0x0a, 0x0d, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x1a, 0x4d, 0x49, 0x47, 0x52, 0x41, 0x54, 0x49, - 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, - 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, - 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x4f, - 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, - 0x41, 0x4e, 0x44, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x10, 0x03, 0x22, 0x56, 0x0a, 0x0d, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x12, 0x1e, 0x0a, 0x1a, 0x54, - 0x41, 0x52, 0x47, 0x45, 0x54, 0x5f, 0x44, 0x49, 0x41, 0x4c, 0x45, 0x43, 0x54, 0x5f, 0x55, 0x4e, - 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x47, - 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x4e, 0x44, 0x41, 0x52, 0x44, 0x5f, 0x53, - 0x51, 0x4c, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x4f, 0x53, 0x54, 0x47, 0x52, 0x45, 0x53, - 0x10, 0x02, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x42, 0x09, 0x0a, 0x07, 0x5f, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x1e, 0x0a, 0x1c, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, 0x63, - 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x6d, 0x69, 0x67, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x5f, 0x64, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x42, 0x12, 0x0a, 0x10, - 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, - 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x69, - 0x7a, 0x65, 0x42, 0x0c, 0x5a, 0x0a, 0x2f, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x62, 0x79, 0x74, + 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x48, 0x06, 0x52, 0x0a, 0x77, 0x72, 0x69, 0x74, + 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x88, 0x01, 0x01, 0x1a, 0xc9, 0x04, 0x0a, 0x0e, 0x53, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x22, 0x0a, 0x0a, + 0x6e, 0x75, 0x6d, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, + 0x48, 0x00, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x88, 0x01, 0x01, + 0x12, 0x35, 0x0a, 0x14, 0x6d, 0x61, 0x78, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, 0x61, + 0x76, 0x65, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x48, 0x01, + 0x52, 0x12, 0x6d, 0x61, 0x78, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, 0x61, 0x76, 0x65, 0x44, + 0x65, 0x70, 0x74, 0x68, 0x88, 0x01, 0x01, 0x12, 0x2c, 0x0a, 0x0f, 0x6e, 0x75, 0x6d, 0x5f, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, 0x61, 0x76, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, + 0x48, 0x02, 0x52, 0x0e, 0x6e, 0x75, 0x6d, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, 0x61, 0x76, + 0x65, 0x73, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x5f, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x48, 0x03, 0x52, 0x0a, 0x6e, 0x75, + 0x6d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x88, 0x01, 0x01, 0x12, 0x2b, 0x0a, 0x0f, 0x6e, + 0x75, 0x6d, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x05, 0x48, 0x04, 0x52, 0x0d, 0x6e, 0x75, 0x6d, 0x50, 0x72, 0x69, 0x6d, 0x61, + 0x72, 0x79, 0x4b, 0x65, 0x79, 0x88, 0x01, 0x01, 0x12, 0x2b, 0x0a, 0x0f, 0x6e, 0x75, 0x6d, 0x5f, + 0x66, 0x6f, 0x72, 0x65, 0x69, 0x67, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x05, 0x48, 0x05, 0x52, 0x0d, 0x6e, 0x75, 0x6d, 0x46, 0x6f, 0x72, 0x65, 0x69, 0x67, 0x6e, 0x4b, + 0x65, 0x79, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x48, 0x06, 0x52, 0x0a, 0x6e, 0x75, + 0x6d, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x88, 0x01, 0x01, 0x12, 0x26, 0x0a, 0x0c, 0x6e, + 0x75, 0x6d, 0x5f, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x05, 0x48, 0x07, 0x52, 0x0b, 0x6e, 0x75, 0x6d, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, + 0x88, 0x01, 0x01, 0x12, 0x33, 0x0a, 0x13, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x70, + 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, + 0x48, 0x08, 0x52, 0x11, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x50, 0x72, 0x69, 0x6d, 0x61, + 0x72, 0x79, 0x4b, 0x65, 0x79, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x6e, 0x75, 0x6d, + 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x6d, 0x61, 0x78, 0x5f, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, 0x61, 0x76, 0x65, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, + 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x65, + 0x61, 0x76, 0x65, 0x73, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x65, 0x73, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x70, 0x72, 0x69, + 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6e, 0x75, 0x6d, + 0x5f, 0x66, 0x6f, 0x72, 0x65, 0x69, 0x67, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x42, 0x0e, 0x0a, 0x0c, + 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x42, 0x0f, 0x0a, 0x0d, + 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x42, 0x16, 0x0a, + 0x14, 0x5f, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, + 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x6e, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, + 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x59, 0x53, 0x51, 0x4c, + 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x50, 0x4f, 0x53, 0x54, 0x47, 0x52, 0x45, 0x53, 0x51, 0x4c, + 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x59, 0x4e, 0x41, 0x4d, 0x4f, 0x44, 0x42, 0x10, 0x03, + 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x51, 0x4c, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x04, + 0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x52, 0x41, 0x43, 0x4c, 0x45, 0x10, 0x05, 0x12, 0x07, 0x0a, 0x03, + 0x43, 0x53, 0x56, 0x10, 0x06, 0x22, 0x76, 0x0a, 0x19, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, + 0x73, 0x6d, 0x12, 0x2b, 0x0a, 0x27, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x43, 0x4f, 0x4e, + 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x43, 0x48, 0x41, 0x4e, 0x49, 0x53, + 0x4d, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, + 0x15, 0x0a, 0x11, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, + 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x42, 0x5f, 0x44, 0x55, 0x4d, + 0x50, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x03, 0x22, 0x64, 0x0a, + 0x0d, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, + 0x0a, 0x1a, 0x4d, 0x49, 0x47, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0f, + 0x0a, 0x0b, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x12, + 0x0d, 0x0a, 0x09, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x12, 0x13, + 0x0a, 0x0f, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, 0x41, 0x4e, 0x44, 0x5f, 0x44, 0x41, 0x54, + 0x41, 0x10, 0x03, 0x22, 0x56, 0x0a, 0x0d, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x44, 0x69, 0x61, + 0x6c, 0x65, 0x63, 0x74, 0x12, 0x1e, 0x0a, 0x1a, 0x54, 0x41, 0x52, 0x47, 0x45, 0x54, 0x5f, 0x44, + 0x49, 0x41, 0x4c, 0x45, 0x43, 0x54, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x47, 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x53, + 0x54, 0x41, 0x4e, 0x44, 0x41, 0x52, 0x44, 0x5f, 0x53, 0x51, 0x4c, 0x10, 0x01, 0x12, 0x0c, 0x0a, + 0x08, 0x50, 0x4f, 0x53, 0x54, 0x47, 0x52, 0x45, 0x53, 0x10, 0x02, 0x42, 0x17, 0x0a, 0x15, 0x5f, + 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x5f, 0x69, 0x64, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, + 0x1e, 0x0a, 0x1c, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x42, + 0x11, 0x0a, 0x0f, 0x5f, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x64, 0x69, + 0x61, 0x6c, 0x65, 0x63, 0x74, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x5f, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x77, 0x72, + 0x69, 0x74, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x42, 0x0c, 0x5a, 0x0a, 0x2f, 0x6d, 0x69, + 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/migration_data.proto b/proto/migration_data.proto index c68af75dd..d55f82dff 100644 --- a/proto/migration_data.proto +++ b/proto/migration_data.proto @@ -4,7 +4,8 @@ option go_package = "/migration"; message MigrationData { - // GUID with prefix ‘HB’ to identify that the request was made via HarbourBridge. + // A system generated id with prefix ‘HB’ to identify that the request was + // made via HarbourBridge. optional string migration_request_id = 1; enum Source { @@ -14,6 +15,7 @@ message MigrationData { DYNAMODB = 3; SQL_SERVER = 4; ORACLE = 5; + CSV = 6; } // Source for the migration. optional Source source = 2; @@ -22,6 +24,7 @@ message MigrationData { SOURCE_CONNECTION_MECHANISM_UNSPECIFIED = 0; DIRECT_CONNECTION = 1; DB_DUMP = 2; + FILE = 3; } // Mechanism used to fetch source data. optional SourceConnectionMechanism source_connection_mechanism = 3; @@ -32,7 +35,7 @@ message MigrationData { DATA_ONLY = 2; SCHEMA_AND_DATA = 3; } - // type of migration + // Type of migration. optional MigrationType migration_type = 4; enum TargetDialect { @@ -40,32 +43,30 @@ message MigrationData { GOOGLE_STANDARD_SQL = 1; POSTGRES = 2; } - // type of target dialect + // Type of target dialect. optional TargetDialect target_dialect = 5; message SchemaPatterns { - // number of tables in the database being migrated + // Number of tables in the database being migrated. optional int32 num_tables = 1; optional int32 max_interleave_depth = 2; - // number of interleaves in the database being migrated + // Number of interleaves in the database being migrated. optional int32 num_interleaves = 3; - // number of indexes in the database being migrated + // Number of indexes in the database being migrated. optional int32 num_indexes = 4; - // number of primary key constraints in the database being migrated + // Number of primary key constraints in the database being migrated. optional int32 num_primary_key = 5; - // number of default constraints in the database being migrated - optional int32 num_default_constraint = 6; - // number of foreign key in the database being migrated - optional int32 num_foreign_key = 7; - // number of columns in the database - optional int32 num_columns = 8; - // number of warnings occurred for the columns that didn’t map cleanly - optional int32 num_warnings = 9; - // indicates whether the source db schema had missing primary key - optional bool missing_primary_key = 10; + // Number of foreign key in the database being migrated. + optional int32 num_foreign_key = 6; + // Number of columns in the database. + optional int32 num_columns = 7; + // Number of warnings occurred for the columns that didn’t map cleanly. + optional int32 num_warnings = 8; + // Indicates whether the source db schema had missing primary key. + optional bool missing_primary_key = 9; } optional SchemaPatterns schema_patterns = 6; - // size of each mutation - it will be populated from the writes_bytes we receive in the request - optional int64 mutation_size = 7; + // Size of each commit in bytes. + optional int64 write_bytes = 7; } From b8df3b357c714d2b9439624c1d3b61e0fe2fca1f Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Wed, 2 Mar 2022 15:35:21 +0530 Subject: [PATCH 10/20] correcting typo --- cmd/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/data.go b/cmd/data.go index d06e12bef..7f1d8c18b 100644 --- a/cmd/data.go +++ b/cmd/data.go @@ -138,7 +138,7 @@ func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface } defer adminClient.Close() - // Populating migration dara in conv. + // Populating migration data in conv. utils.PopulateMigrationData(conv, sourceProfile.Driver, targetProfile.TargetDb) conv.MigrationData.MigrationType = migration.MigrationData_DATA_ONLY.Enum() conversion.Report(sourceProfile.Driver, nil, ioHelper.BytesRead, "", conv, cmd.filePrefix+reportFile, ioHelper.Out) From 0ec6c8a807b7e44969d916a4812c175158474643 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Wed, 2 Mar 2022 16:29:20 +0530 Subject: [PATCH 11/20] fixing tests --- cmd/schema_and_data.go | 2 +- internal/report.go | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/schema_and_data.go b/cmd/schema_and_data.go index 4426a9e62..b679d4c86 100644 --- a/cmd/schema_and_data.go +++ b/cmd/schema_and_data.go @@ -111,7 +111,7 @@ func (cmd *SchemaAndDataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ... panic(err) } - // Populating migration dara in conv. + // Populating migration data in conv. utils.PopulateMigrationData(conv, sourceProfile.Driver, targetProfile.TargetDb) conv.MigrationData.MigrationType = migration.MigrationData_SCHEMA_AND_DATA.Enum() diff --git a/internal/report.go b/internal/report.go index 3b642ab37..4ad35eddf 100644 --- a/internal/report.go +++ b/internal/report.go @@ -396,7 +396,9 @@ func GenerateSummary(conv *Conv, r []tableReport, badWrites map[string]int64) st warnings += t.Warnings * weight if t.SyntheticPKey != "" { missingPKey = true - conv.MigrationData.SchemaPatterns.MissingPrimaryKey = &missingPKey + if conv.MigrationData.GetSchemaPatterns().GetMissingPrimaryKey() { + conv.MigrationData.SchemaPatterns.MissingPrimaryKey = &missingPKey + } } } // Don't use tableReport for rows/badRows stats because tableReport @@ -411,8 +413,10 @@ func GenerateSummary(conv *Conv, r []tableReport, badWrites map[string]int64) st } numColumns := int32(int(cols)) numWarnings := int32(int(warnings)) - conv.MigrationData.SchemaPatterns.NumColumns = &numColumns - conv.MigrationData.SchemaPatterns.NumWarnings = &numWarnings + if conv.MigrationData.GetSchemaPatterns() != nil { + conv.MigrationData.SchemaPatterns.NumColumns = &numColumns + conv.MigrationData.SchemaPatterns.NumWarnings = &numWarnings + } return rateConversion(rows, badRows, cols, warnings, missingPKey, true, conv.SchemaMode()) } From a16c4e709014591d6126dbd80b1b2098a61bf5f7 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Wed, 2 Mar 2022 17:03:27 +0530 Subject: [PATCH 12/20] made changes to cmd.go --- cmd/cmd.go | 11 +++++++++++ go.mod | 1 - 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index a577ca5dd..5921a88d8 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -27,6 +27,7 @@ import ( "github.com/cloudspannerecosystem/harbourbridge/conversion" "github.com/cloudspannerecosystem/harbourbridge/internal" "github.com/cloudspannerecosystem/harbourbridge/profiles" + "github.com/cloudspannerecosystem/harbourbridge/proto/migration" ) var ( @@ -85,6 +86,16 @@ func CommandLine(ctx context.Context, driver, targetDb, dbURI string, dataOnly, return err } } + + // Populating migration data in conv. + utils.PopulateMigrationData(conv, driver, targetDb) + if dataOnly { + conv.MigrationData.MigrationType = migration.MigrationData_DATA_ONLY.Enum() + } else { + conv.MigrationData.MigrationType = migration.MigrationData_SCHEMA_AND_DATA.Enum() + } + conversion.Report(driver, nil, ioHelper.BytesRead, "", conv, outputFilePrefix+reportFile, ioHelper.Out) + adminClient, err := utils.NewDatabaseAdminClient(ctx) if err != nil { return fmt.Errorf("can't create admin client: %w", utils.AnalyzeError(err, dbURI)) diff --git a/go.mod b/go.mod index f4ada8725..148b265fd 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/go-sql-driver/mysql v1.5.0 - github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.6 github.com/google/subcommands v1.2.0 github.com/gorilla/handlers v1.5.0 From 2a9f2b0f591545d3ec08fd6777f0a873e88bf674 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Thu, 3 Mar 2022 23:51:56 +0530 Subject: [PATCH 13/20] go mod tidy --- go.mod | 10 ++++++---- go.sum | 4 ++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 148b265fd..70b3ed419 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,10 @@ require ( cloud.google.com/go/storage v1.17.0 github.com/DATA-DOG/go-sqlmock v1.4.1 github.com/aws/aws-sdk-go v1.34.5 - github.com/basgys/goxml2json v1.1.0 // indirect - github.com/denisenkom/go-mssqldb v0.11.0 // indirect + github.com/basgys/goxml2json v1.1.0 + github.com/bitly/go-simplejson v0.5.0 // indirect + github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect + github.com/denisenkom/go-mssqldb v0.11.0 github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/go-sql-driver/mysql v1.5.0 github.com/google/go-cmp v0.5.6 @@ -30,8 +32,8 @@ require ( golang.org/x/tools v0.1.7 // indirect google.golang.org/api v0.57.0 google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 - google.golang.org/grpc v1.40.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/grpc v1.40.0 + google.golang.org/protobuf v1.27.1 honnef.co/go/tools v0.2.1 // indirect ) diff --git a/go.sum b/go.sum index 9678b4f1b..e9d903136 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,12 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= From 5190f132e6e4d3836482ceee093121f77fa96a24 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Fri, 4 Mar 2022 00:00:07 +0530 Subject: [PATCH 14/20] experimenting with go.mod file --- go.mod | 11 ++++------- go.sum | 4 ---- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 70b3ed419..802aa84f1 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,8 @@ require ( cloud.google.com/go/storage v1.17.0 github.com/DATA-DOG/go-sqlmock v1.4.1 github.com/aws/aws-sdk-go v1.34.5 - github.com/basgys/goxml2json v1.1.0 - github.com/bitly/go-simplejson v0.5.0 // indirect - github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect - github.com/denisenkom/go-mssqldb v0.11.0 + github.com/basgys/goxml2json v1.1.0 // indirect + github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/go-sql-driver/mysql v1.5.0 github.com/google/go-cmp v0.5.6 @@ -32,9 +30,8 @@ require ( golang.org/x/tools v0.1.7 // indirect google.golang.org/api v0.57.0 google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 - google.golang.org/grpc v1.40.0 - google.golang.org/protobuf v1.27.1 honnef.co/go/tools v0.2.1 // indirect + google.golang.org/protobuf v1.27.1 // indirect ) // cloud.google.com/go will upgrade grpc to v1.40.0 @@ -53,4 +50,4 @@ replace google.golang.org/genproto => github.com/hengfengli/go-genproto v0.0.0-2 // jwt-go, an indirect dependency imported through pingcap/tidb, is an unmaintained repo which has security issues. // Replacing it with the fork that tidb's latest master uses and has the security patch. -replace github.com/dgrijalva/jwt-go => github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible +replace github.com/dgrijalva/jwt-go => github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible \ No newline at end of file diff --git a/go.sum b/go.sum index e9d903136..9678b4f1b 100644 --- a/go.sum +++ b/go.sum @@ -42,12 +42,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= From a6f0f7d0f3a2ba4ffece916ddcfc9aca4dc08b99 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Fri, 4 Mar 2022 00:31:42 +0530 Subject: [PATCH 15/20] go mod modified --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 802aa84f1..14280b0c9 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 honnef.co/go/tools v0.2.1 // indirect google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/grpc v1.40.0 ) // cloud.google.com/go will upgrade grpc to v1.40.0 From edb67ddfd0d49b4768714ceaf7da1d73bb5076b7 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Mon, 7 Mar 2022 11:23:51 +0530 Subject: [PATCH 16/20] minor changes --- common/utils/utils.go | 92 +++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/common/utils/utils.go b/common/utils/utils.go index 4cb4c9758..204ae167b 100644 --- a/common/utils/utils.go +++ b/common/utils/utils.go @@ -509,43 +509,17 @@ func randomString(n int) string { return sb.String() } +// PopulateMigrationData populates migration data like source schema details, +// request id, target dialect, connection mechanism etc in conv object func PopulateMigrationData(conv *internal.Conv, driver, targetDb string) { migrationRequestId := "HB" + randomString(14) - numTables := int32(len(conv.SrcSchema)) migrationData := migration.MigrationData{ MigrationRequestId: &migrationRequestId, - SchemaPatterns: &migration.MigrationData_SchemaPatterns{ - NumTables: &numTables, - }, - } - switch driver { - case constants.PGDUMP: - migrationData.SourceConnectionMechanism = migration.MigrationData_DB_DUMP.Enum() - migrationData.Source = migration.MigrationData_POSTGRESQL.Enum() - case constants.MYSQLDUMP: - migrationData.SourceConnectionMechanism = migration.MigrationData_DB_DUMP.Enum() - migrationData.Source = migration.MigrationData_MYSQL.Enum() - case constants.POSTGRES: - migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() - migrationData.Source = migration.MigrationData_POSTGRESQL.Enum() - case constants.MYSQL: - migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() - migrationData.Source = migration.MigrationData_MYSQL.Enum() - case constants.DYNAMODB: - migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() - migrationData.Source = migration.MigrationData_DYNAMODB.Enum() - case constants.ORACLE: - migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() - migrationData.Source = migration.MigrationData_ORACLE.Enum() - case constants.SQLSERVER: - migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() - migrationData.Source = migration.MigrationData_SQL_SERVER.Enum() - case constants.CSV: - migrationData.SourceConnectionMechanism = migration.MigrationData_FILE.Enum() - migrationData.Source = migration.MigrationData_CSV.Enum() } + populateMigrationDataSourceDetails(driver, &migrationData) + populateMigrationDataSchemaPatterns(conv, &migrationData) switch targetDb { case constants.TargetSpanner: @@ -553,7 +527,14 @@ func PopulateMigrationData(conv *internal.Conv, driver, targetDb string) { case constants.TargetExperimentalPostgres: migrationData.TargetDialect = migration.MigrationData_POSTGRES.Enum() } + conv.MigrationData = migrationData +} +// populateMigrationDataSchemaPatterns populates schema petterns like number of tables, foreign key, primary key, +// indexes, interleaves, max interleave depth and if source schema is missing primary key in migrationData object +func populateMigrationDataSchemaPatterns(conv *internal.Conv, migrationData *migration.MigrationData) { + + numTables := int32(len(conv.SrcSchema)) var numForeignKey, numIndexes, numPrimaryKey, numInterleaves, maxInterleaveDepth int32 = 0, 0, 0, 0, 0 missingPrimaryKey := false @@ -570,11 +551,6 @@ func PopulateMigrationData(conv *internal.Conv, driver, targetDb string) { for _, table := range conv.SpSchema { if table.Parent != "" { numInterleaves++ - } - } - - for _, table := range conv.SpSchema { - if table.Parent != "" { depth := 1 parentTableName := table.Parent for conv.SpSchema[parentTableName].Parent != "" { @@ -585,11 +561,43 @@ func PopulateMigrationData(conv *internal.Conv, driver, targetDb string) { } } - migrationData.SchemaPatterns.NumForeignKey = &numForeignKey - migrationData.SchemaPatterns.NumPrimaryKey = &numPrimaryKey - migrationData.SchemaPatterns.NumIndexes = &numIndexes - migrationData.SchemaPatterns.NumInterleaves = &numInterleaves - migrationData.SchemaPatterns.MissingPrimaryKey = &missingPrimaryKey - migrationData.SchemaPatterns.MaxInterleaveDepth = &maxInterleaveDepth - conv.MigrationData = migrationData + migrationData.SchemaPatterns = &migration.MigrationData_SchemaPatterns{ + NumTables: &numTables, + NumForeignKey: &numForeignKey, + NumInterleaves: &numInterleaves, + MissingPrimaryKey: &missingPrimaryKey, + MaxInterleaveDepth: &maxInterleaveDepth, + NumIndexes: &numIndexes, + } +} + +// populateMigrationDataSourceDetails populates source database type and +// source connection mechanism in migrationData object +func populateMigrationDataSourceDetails(driver string, migrationData *migration.MigrationData) { + switch driver { + case constants.PGDUMP: + migrationData.SourceConnectionMechanism = migration.MigrationData_DB_DUMP.Enum() + migrationData.Source = migration.MigrationData_POSTGRESQL.Enum() + case constants.MYSQLDUMP: + migrationData.SourceConnectionMechanism = migration.MigrationData_DB_DUMP.Enum() + migrationData.Source = migration.MigrationData_MYSQL.Enum() + case constants.POSTGRES: + migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() + migrationData.Source = migration.MigrationData_POSTGRESQL.Enum() + case constants.MYSQL: + migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() + migrationData.Source = migration.MigrationData_MYSQL.Enum() + case constants.DYNAMODB: + migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() + migrationData.Source = migration.MigrationData_DYNAMODB.Enum() + case constants.ORACLE: + migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() + migrationData.Source = migration.MigrationData_ORACLE.Enum() + case constants.SQLSERVER: + migrationData.SourceConnectionMechanism = migration.MigrationData_DIRECT_CONNECTION.Enum() + migrationData.Source = migration.MigrationData_SQL_SERVER.Enum() + case constants.CSV: + migrationData.SourceConnectionMechanism = migration.MigrationData_FILE.Enum() + migrationData.Source = migration.MigrationData_CSV.Enum() + } } From 802e46f4545e69a78cb9a46dbd103d441047f949 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Mon, 7 Mar 2022 11:30:55 +0530 Subject: [PATCH 17/20] testing 1 --- go.mod | 2 +- go.sum | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 14280b0c9..c44481699 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( // cloud.google.com/go will upgrade grpc to v1.40.0 // We need keep the replacement since google.golang.org/grpc/naming isn't // available in higher versions. -replace google.golang.org/grpc => google.golang.org/grpc v1.29.1 +replace google.golang.org/grpc => google.golang.org/grpc v1.40.0 // Temporary workaround until PG support is available in // cloud.google.com/go/spanner public release. diff --git a/go.sum b/go.sum index 9678b4f1b..35adb076d 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpTwxyUrC2DNw0= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/aws/aws-sdk-go v1.34.5 h1:FwubVVX9u+kW9qDCjVzyWOdsL+W5wPq683wMk2R2GXk= @@ -51,6 +52,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -93,6 +96,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -190,6 +194,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -207,6 +212,7 @@ github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -227,6 +233,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtguw7vR+nGtnDjY= github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -480,6 +487,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= @@ -706,6 +714,8 @@ google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -714,6 +724,7 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= From 7e09d4bf591b24fdb8516f8106dfe92536165093 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Mon, 7 Mar 2022 11:42:47 +0530 Subject: [PATCH 18/20] commented metadata --- conversion/conversion.go | 25 ++++++++++++++----------- go.mod | 4 ++-- go.sum | 2 -- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/conversion/conversion.go b/conversion/conversion.go index 0cc2d3234..a11d92dd4 100644 --- a/conversion/conversion.go +++ b/conversion/conversion.go @@ -43,13 +43,13 @@ import ( "github.com/aws/aws-sdk-go/aws/session" dydb "github.com/aws/aws-sdk-go/service/dynamodb" adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" - "google.golang.org/grpc/metadata" + + //"google.golang.org/grpc/metadata" "github.com/cloudspannerecosystem/harbourbridge/common/constants" "github.com/cloudspannerecosystem/harbourbridge/common/utils" "github.com/cloudspannerecosystem/harbourbridge/internal" "github.com/cloudspannerecosystem/harbourbridge/profiles" - "github.com/cloudspannerecosystem/harbourbridge/proto/migration" "github.com/cloudspannerecosystem/harbourbridge/sources/common" "github.com/cloudspannerecosystem/harbourbridge/sources/csv" "github.com/cloudspannerecosystem/harbourbridge/sources/dynamodb" @@ -192,11 +192,12 @@ func dataFromDatabase(sourceProfile profiles.SourceProfile, config writer.BatchW p := internal.NewProgress(totalRows, "Writing data to Spanner", internal.Verbose(), false) rows := int64(0) config.Write = func(m []*sp.Mutation) error { - migrationData := migration.MigrationData{ + /*migrationData := migration.MigrationData{ MigrationRequestId: conv.MigrationData.MigrationRequestId, } - migrationMetadataValue := migrationData.String() - _, err := client.Apply(metadata.AppendToOutgoingContext(context.Background(), migrationMetadataKey, migrationMetadataValue), m) + //migrationMetadataValue := migrationData.String() + _, err := client.Apply(metadata.AppendToOutgoingContext(context.Background(), migrationMetadataKey, migrationMetadataValue), m)*/ + _, err := client.Apply(context.Background(), m) if err != nil { return err } @@ -276,11 +277,12 @@ func dataFromDump(driver string, config writer.BatchWriterConfig, ioHelper *util r := internal.NewReader(bufio.NewReader(ioHelper.SeekableIn), nil) rows := int64(0) config.Write = func(m []*sp.Mutation) error { - migrationData := migration.MigrationData{ + /*migrationData := migration.MigrationData{ MigrationRequestId: conv.MigrationData.MigrationRequestId, } migrationMetadataValue := migrationData.String() - _, err := client.Apply(metadata.AppendToOutgoingContext(context.Background(), migrationMetadataKey, migrationMetadataValue), m) + _, err := client.Apply(metadata.AppendToOutgoingContext(context.Background(), migrationMetadataKey, migrationMetadataValue), m)*/ + _, err := client.Apply(context.Background(), m) if err != nil { return err } @@ -344,11 +346,12 @@ func dataFromCSV(ctx context.Context, sourceProfile profiles.SourceProfile, targ p := internal.NewProgress(totalRows, "Writing data to Spanner", internal.Verbose(), false) rows := int64(0) config.Write = func(m []*sp.Mutation) error { - migrationData := migration.MigrationData{ + /*migrationData := migration.MigrationData{ MigrationRequestId: conv.MigrationData.MigrationRequestId, } migrationMetadataValue := migrationData.String() - _, err := client.Apply(metadata.AppendToOutgoingContext(context.Background(), migrationMetadataKey, migrationMetadataValue), m) + _, err := client.Apply(metadata.AppendToOutgoingContext(context.Background(), migrationMetadataKey, migrationMetadataValue), m)*/ + _, err := client.Apply(context.Background(), m) if err != nil { return err } @@ -483,8 +486,8 @@ func CreateOrUpdateDatabase(ctx context.Context, adminClient *database.DatabaseA return err } // Adding migration metadata to the outgoing context. - migrationMetadataValue := conv.MigrationData.String() - ctx = metadata.AppendToOutgoingContext(ctx, migrationMetadataKey, migrationMetadataValue) + //migrationMetadataValue := conv.MigrationData.String() + //ctx = metadata.AppendToOutgoingContext(ctx, migrationMetadataKey, migrationMetadataValue) if dbExists { err := UpdateDatabase(ctx, adminClient, dbURI, conv, out) if err != nil { diff --git a/go.mod b/go.mod index c44481699..0496d2952 100644 --- a/go.mod +++ b/go.mod @@ -32,13 +32,13 @@ require ( google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 honnef.co/go/tools v0.2.1 // indirect google.golang.org/protobuf v1.27.1 // indirect - google.golang.org/grpc v1.40.0 + //google.golang.org/grpc v1.40.0 ) // cloud.google.com/go will upgrade grpc to v1.40.0 // We need keep the replacement since google.golang.org/grpc/naming isn't // available in higher versions. -replace google.golang.org/grpc => google.golang.org/grpc v1.40.0 +replace google.golang.org/grpc => google.golang.org/grpc v1.29.1 // Temporary workaround until PG support is available in // cloud.google.com/go/spanner public release. diff --git a/go.sum b/go.sum index 35adb076d..df43a2254 100644 --- a/go.sum +++ b/go.sum @@ -714,8 +714,6 @@ google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 250bafbb038261f49d4e6cd71fe14fb296485f51 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Mon, 7 Mar 2022 11:46:12 +0530 Subject: [PATCH 19/20] commented metadata --- go.mod | 12 +++++++----- go.sum | 13 ++++--------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 0496d2952..8349678f4 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,10 @@ require ( cloud.google.com/go/storage v1.17.0 github.com/DATA-DOG/go-sqlmock v1.4.1 github.com/aws/aws-sdk-go v1.34.5 - github.com/basgys/goxml2json v1.1.0 // indirect - github.com/denisenkom/go-mssqldb v0.11.0 // indirect + github.com/basgys/goxml2json v1.1.0 + github.com/bitly/go-simplejson v0.5.0 // indirect + github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect + github.com/denisenkom/go-mssqldb v0.11.0 github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/go-sql-driver/mysql v1.5.0 github.com/google/go-cmp v0.5.6 @@ -30,9 +32,9 @@ require ( golang.org/x/tools v0.1.7 // indirect google.golang.org/api v0.57.0 google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 + google.golang.org/protobuf v1.27.1 honnef.co/go/tools v0.2.1 // indirect - google.golang.org/protobuf v1.27.1 // indirect - //google.golang.org/grpc v1.40.0 +//google.golang.org/grpc v1.40.0 ) // cloud.google.com/go will upgrade grpc to v1.40.0 @@ -51,4 +53,4 @@ replace google.golang.org/genproto => github.com/hengfengli/go-genproto v0.0.0-2 // jwt-go, an indirect dependency imported through pingcap/tidb, is an unmaintained repo which has security issues. // Replacing it with the fork that tidb's latest master uses and has the security patch. -replace github.com/dgrijalva/jwt-go => github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible \ No newline at end of file +replace github.com/dgrijalva/jwt-go => github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible diff --git a/go.sum b/go.sum index df43a2254..e9d903136 100644 --- a/go.sum +++ b/go.sum @@ -32,7 +32,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpTwxyUrC2DNw0= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/aws/aws-sdk-go v1.34.5 h1:FwubVVX9u+kW9qDCjVzyWOdsL+W5wPq683wMk2R2GXk= @@ -43,8 +42,12 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -52,8 +55,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -96,7 +97,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -194,7 +194,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -212,7 +211,6 @@ github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -233,7 +231,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtguw7vR+nGtnDjY= github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -487,7 +484,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= @@ -722,7 +718,6 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= From 6a6c33beb81f10085cf482f8ad2c0d10bec899b0 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Mon, 7 Mar 2022 12:32:21 +0530 Subject: [PATCH 20/20] fixed pb file --- proto/migration/migration_data.pb.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proto/migration/migration_data.pb.go b/proto/migration/migration_data.pb.go index c2ee5b8ab..fcf599008 100644 --- a/proto/migration/migration_data.pb.go +++ b/proto/migration/migration_data.pb.go @@ -7,10 +7,11 @@ package migration import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const (