diff --git a/pkg/dbutil/retry.go b/pkg/dbutil/retry.go index 5949c2c9b..4c2784750 100644 --- a/pkg/dbutil/retry.go +++ b/pkg/dbutil/retry.go @@ -26,6 +26,8 @@ var ( Retryable1105Msgs = []string{ "Information schema is out of date", "Information schema is changed", + "context deadline exceeded", + "exceeding the deadline", } ) diff --git a/sync_diff_inspector/source/mysql_shard.go b/sync_diff_inspector/source/mysql_shard.go index 004b8195e..23d5d3e59 100644 --- a/sync_diff_inspector/source/mysql_shard.go +++ b/sync_diff_inspector/source/mysql_shard.go @@ -316,7 +316,7 @@ func NewMySQLSources(ctx context.Context, tableDiffs []*common.TableDiff, ds []* if sourceDB.Router != nil { targetSchema, targetTable, err = sourceDB.Router.Route(schema, table) if err != nil { - return nil, errors.Errorf("get route result for %d source %s.%s failed, error %v", i, schema, table, err) + return nil, errors.Errorf("get route result for %d source %s.%s failed, please check the rules of schema/table routes, error %v", i, schema, table, err) } } uniqueId := utils.UniqueID(targetSchema, targetTable) diff --git a/sync_diff_inspector/source/tidb.go b/sync_diff_inspector/source/tidb.go index 842cb6d6a..209a1ae61 100644 --- a/sync_diff_inspector/source/tidb.go +++ b/sync_diff_inspector/source/tidb.go @@ -228,7 +228,7 @@ func getSourceTableMap(ctx context.Context, tableDiffs []*common.TableDiff, ds * if ds.Router != nil { targetSchema, targetTable, err = ds.Router.Route(schema, table) if err != nil { - return nil, errors.Errorf("get route result for %s.%s failed, error %v", schema, table, err) + return nil, errors.Errorf("get route result for %s.%s failed, please check the rules of schema/table routes, error %v", schema, table, err) } } diff --git a/sync_diff_inspector/utils/utils.go b/sync_diff_inspector/utils/utils.go index 78cfccfd4..c9aaa3548 100644 --- a/sync_diff_inspector/utils/utils.go +++ b/sync_diff_inspector/utils/utils.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" "sync" + "time" "github.com/olekukonko/tablewriter" "github.com/pingcap/errors" @@ -741,10 +742,23 @@ func GetCountAndCRC32Checksum(ctx context.Context, db *sql.DB, schemaName, table var count sql.NullInt64 var checksum sql.NullInt64 - err := db.QueryRowContext(ctx, query, args...).Scan(&count, &checksum) - if err != nil { - log.Warn("execute checksum query fail", zap.String("query", query), zap.Reflect("args", args), zap.Error(err)) - return -1, -1, errors.Trace(err) + var err error + for i := 0; i < dbutil.DefaultRetryTime; i++ { + err = db.QueryRowContext(ctx, query, args...).Scan(&count, &checksum) + if err != nil { + if dbutil.IsRetryableError(err) && i < dbutil.DefaultRetryTime-1 { + select { + case <-ctx.Done(): + return -1, -1, errors.Trace(ctx.Err()) + case <-time.After(10 * time.Millisecond): + } + log.Warn("execute checksum query timeout, retry...", zap.Int("retry count", i+1)) + continue + } + log.Warn("execute checksum query fail", zap.String("query", query), zap.Reflect("args", args), zap.Error(err)) + return -1, -1, errors.Trace(err) + } + break } if !count.Valid || !checksum.Valid { // if don't have any data, the checksum will be `NULL`