diff: add config to only use checksum, skip select data (#215) (#226)

pkg/diff/diff.go

@@ -74,6 +74,9 @@ type TableDiff struct {
 	// set false if want to comapre the data directly
 	UseChecksum bool
 
+	// set true if just want compare data by checksum, will skip select data when checksum is not equal
+	OnlyUseChecksum bool
+
 	// collation config in mysql/tidb, should corresponding to charset.
 	Collation string
 
@@ -284,6 +287,11 @@ func (t *TableDiff) checkChunkDataEqual(ctx context.Context, checkJobs []*CheckJ
 			log.Warnf("table: %s, range: %s, args: %v, checksum is not equal, one is %d, another is %d", dbutil.TableName(job.Schema, job.Table), job.Where, job.Args, sourceChecksum, targetChecksum)
 		}
 
+		if t.UseChecksum && t.OnlyUseChecksum {
+			equal = false
+			continue
+		}
+
 		// if checksum is not equal or don't need compare checksum, compare the data
 		log.Infof("select data from %s for range (where: %s, args: %v) and then check data", dbutil.TableName(job.Schema, job.Table), job.Where, job.Args)
 		sourceRows := make(map[string][]map[string]*dbutil.ColumnData)

sync_diff_inspector/config.go

@@ -180,6 +180,9 @@ type Config struct {
 	// set false if want to comapre the data directly
 	UseChecksum bool `toml:"use-checksum" json:"use-checksum"`
 
+	// set true if just want compare data by checksum, will skip select data when checksum is not equal.
+	OnlyUseChecksum bool `toml:"only-use-checksum" json:"only-use-checksum"`
+
 	// the name of the file which saves sqls used to fix different data
 	FixSQLFile string `toml:"fix-sql-file" json:"fix-sql-file"`
 
@@ -312,5 +315,17 @@ func (c *Config) checkConfig() bool {
 		}
 	}
 
+	if c.OnlyUseChecksum {
+		if !c.UseChecksum {
+			log.Error("need set use-checksum = true")
+			return false
+		}
+	} else {
+		if len(c.FixSQLFile) == 0 {
+			log.Warn("fix-sql-file is invalid, will use default value 'fix.sql'")
+			c.FixSQLFile = "fix.sql"
+		}
+	}
+
 	return true
 }

sync_diff_inspector/config.toml

@@ -18,15 +18,19 @@ sample-percent = 100
 use-rowid = false
 
 # calculate the data's checksum, and compare data by checksum.
+# set false if want to comapre the data directly
 use-checksum = true
 
+# set true if just want compare data by checksum, will skip select data when checksum is not equal. 
+only-use-checksum = false
+
 # ignore check table's data
 ignore-data-check = false
 
 # ignore check table's struct
 ignore-struct-check = false
 
-# the name of the file which saves sqls used to fix different data
+# the name of the file which saves sqls used to fix different data.
 fix-sql-file = "fix.sql"
 
 # use this tidb's statistics information to split chunk

sync_diff_inspector/diff.go

@@ -36,6 +36,7 @@ type Diff struct {
 	checkThreadCount  int
 	useRowID          bool
 	useChecksum       bool
+	onlyUseChecksum   bool
 	ignoreDataCheck   bool
 	ignoreStructCheck bool
 	tables            map[string]map[string]*TableConfig
@@ -56,6 +57,7 @@ func NewDiff(ctx context.Context, cfg *Config) (diff *Diff, err error) {
 		checkThreadCount:  cfg.CheckThreadCount,
 		useRowID:          cfg.UseRowID,
 		useChecksum:       cfg.UseChecksum,
+		onlyUseChecksum:   cfg.OnlyUseChecksum,
 		ignoreDataCheck:   cfg.IgnoreDataCheck,
 		ignoreStructCheck: cfg.IgnoreStructCheck,
 		tidbInstanceID:    cfg.TiDBInstanceID,
@@ -404,6 +406,7 @@ func (df *Diff) Equal() (err error) {
 				CheckThreadCount:  df.checkThreadCount,
 				UseRowID:          df.useRowID,
 				UseChecksum:       df.useChecksum,
+				OnlyUseChecksum:   df.onlyUseChecksum,
 				IgnoreStructCheck: df.ignoreStructCheck,
 				IgnoreDataCheck:   df.ignoreDataCheck,
 				TiDBStatsSource:   tidbStatsSource,