From 51f26fc27f6213bc1a3e948697cb520d1e9173b4 Mon Sep 17 00:00:00 2001 From: Leavrth Date: Wed, 21 Sep 2022 11:08:12 +0800 Subject: [PATCH 1/4] add retry for checksum Signed-off-by: Leavrth --- sync_diff_inspector/utils/utils.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/sync_diff_inspector/utils/utils.go b/sync_diff_inspector/utils/utils.go index 78cfccfd4..c9aaa3548 100644 --- a/sync_diff_inspector/utils/utils.go +++ b/sync_diff_inspector/utils/utils.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" "sync" + "time" "github.com/olekukonko/tablewriter" "github.com/pingcap/errors" @@ -741,10 +742,23 @@ func GetCountAndCRC32Checksum(ctx context.Context, db *sql.DB, schemaName, table var count sql.NullInt64 var checksum sql.NullInt64 - err := db.QueryRowContext(ctx, query, args...).Scan(&count, &checksum) - if err != nil { - log.Warn("execute checksum query fail", zap.String("query", query), zap.Reflect("args", args), zap.Error(err)) - return -1, -1, errors.Trace(err) + var err error + for i := 0; i < dbutil.DefaultRetryTime; i++ { + err = db.QueryRowContext(ctx, query, args...).Scan(&count, &checksum) + if err != nil { + if dbutil.IsRetryableError(err) && i < dbutil.DefaultRetryTime-1 { + select { + case <-ctx.Done(): + return -1, -1, errors.Trace(ctx.Err()) + case <-time.After(10 * time.Millisecond): + } + log.Warn("execute checksum query timeout, retry...", zap.Int("retry count", i+1)) + continue + } + log.Warn("execute checksum query fail", zap.String("query", query), zap.Reflect("args", args), zap.Error(err)) + return -1, -1, errors.Trace(err) + } + break } if !count.Valid || !checksum.Valid { // if don't have any data, the checksum will be `NULL` From 4e9fc4e4ea08183f984102559ebce6b9032a2e53 Mon Sep 17 00:00:00 2001 From: Leavrth Date: Wed, 21 Sep 2022 11:10:18 +0800 Subject: [PATCH 2/4] retry for 1105 context deadline exceeded Signed-off-by: Leavrth --- pkg/dbutil/retry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/dbutil/retry.go b/pkg/dbutil/retry.go index 5949c2c9b..b600616b0 100644 --- a/pkg/dbutil/retry.go +++ b/pkg/dbutil/retry.go @@ -26,6 +26,7 @@ var ( Retryable1105Msgs = []string{ "Information schema is out of date", "Information schema is changed", + "context deadline exceeded", } ) From c6a046793b90658d38727af21e19e2dfd8c8d4ce Mon Sep 17 00:00:00 2001 From: Leavrth Date: Thu, 22 Sep 2022 14:02:07 +0800 Subject: [PATCH 3/4] make error hint more readable Signed-off-by: Leavrth --- sync_diff_inspector/source/mysql_shard.go | 2 +- sync_diff_inspector/source/tidb.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sync_diff_inspector/source/mysql_shard.go b/sync_diff_inspector/source/mysql_shard.go index 004b8195e..23d5d3e59 100644 --- a/sync_diff_inspector/source/mysql_shard.go +++ b/sync_diff_inspector/source/mysql_shard.go @@ -316,7 +316,7 @@ func NewMySQLSources(ctx context.Context, tableDiffs []*common.TableDiff, ds []* if sourceDB.Router != nil { targetSchema, targetTable, err = sourceDB.Router.Route(schema, table) if err != nil { - return nil, errors.Errorf("get route result for %d source %s.%s failed, error %v", i, schema, table, err) + return nil, errors.Errorf("get route result for %d source %s.%s failed, please check the rules of schema/table routes, error %v", i, schema, table, err) } } uniqueId := utils.UniqueID(targetSchema, targetTable) diff --git a/sync_diff_inspector/source/tidb.go b/sync_diff_inspector/source/tidb.go index 842cb6d6a..209a1ae61 100644 --- a/sync_diff_inspector/source/tidb.go +++ b/sync_diff_inspector/source/tidb.go @@ -228,7 +228,7 @@ func getSourceTableMap(ctx context.Context, tableDiffs []*common.TableDiff, ds * if ds.Router != nil { targetSchema, targetTable, err = ds.Router.Route(schema, table) if err != nil { - return nil, errors.Errorf("get route result for %s.%s failed, error %v", schema, table, err) + return nil, errors.Errorf("get route result for %s.%s failed, please check the rules of schema/table routes, error %v", schema, table, err) } } From 65d3beef35d760930e654e78986ac4105f1b8767 Mon Sep 17 00:00:00 2001 From: Leavrth Date: Tue, 17 Jan 2023 11:11:16 +0800 Subject: [PATCH 4/4] add a new retry error msg Signed-off-by: Leavrth --- pkg/dbutil/retry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/dbutil/retry.go b/pkg/dbutil/retry.go index b600616b0..4c2784750 100644 --- a/pkg/dbutil/retry.go +++ b/pkg/dbutil/retry.go @@ -27,6 +27,7 @@ var ( "Information schema is out of date", "Information schema is changed", "context deadline exceeded", + "exceeding the deadline", } )