*: build sync_diff_inspector from source code to run tests (#11988)

close #11672

.github/workflows/dm_binlog_999999.yaml

@@ -64,8 +64,6 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y curl
           sudo docker-compose -f ./dm/tests/binlog_999999/docker-compose.yml up -d
-          curl http://download.pingcap.org/tidb-enterprise-tools-nightly-linux-amd64.tar.gz | tar xz
-          mv tidb-enterprise-tools-nightly-linux-amd64/bin/sync_diff_inspector bin/
           curl http://download.pingcap.org/tidb-nightly-linux-amd64.tar.gz | tar xz
           mv tidb-nightly-linux-amd64/bin/tidb-server bin/
           curl -O https://dl.min.io/server/minio/release/linux-amd64/minio

Makefile

@@ -6,6 +6,7 @@
 	mysql_docker_integration_test mysql_docker_integration_test_with_build \
 	build_mysql_integration_test_images clean_integration_test_images \
 	dm dm-master dm-worker dmctl dm-syncer dm_coverage \
+	sync-diff-inspector \
 	engine tiflow tiflow-demo tiflow-chaos-case engine_image help \
 	format-makefiles check-makefiles oauth2_server prepare_test_binaries
 
@@ -219,14 +220,13 @@ check_third_party_binary:
 	@which bin/pd-server
 	@which bin/tiflash
 	@which bin/pd-ctl
-	@which bin/sync_diff_inspector
 	@which bin/go-ycsb
 	@which bin/etcdctl
 	@which bin/jq
 	@which bin/minio
 	@which bin/bin/schema-registry-start
 
-integration_test_build: check_failpoint_ctl storage_consumer kafka_consumer pulsar_consumer oauth2_server
+integration_test_build: check_failpoint_ctl storage_consumer kafka_consumer pulsar_consumer oauth2_server sync-diff-inspector
 	$(FAILPOINT_ENABLE)
 	$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
 		-coverpkg=github.com/pingcap/tiflow/... \
@@ -253,7 +253,7 @@ build_mysql_integration_test_images: ## Build MySQL integration test images with
 build_mysql_integration_test_images: clean_integration_test_containers
 	docker-compose -f $(TICDC_DOCKER_DEPLOYMENTS_DIR)/docker-compose-mysql-integration.yml build --no-cache
 
-integration_test_kafka: check_third_party_binary
+integration_test_kafka: check_third_party_binary sync-diff-inspector
 	tests/integration_tests/run.sh kafka "$(CASE)" "$(START_AT)"
 
 integration_test_storage:
@@ -385,6 +385,9 @@ clean:
 	rm -rf tools/bin
 	rm -rf tools/include
 
+sync-diff-inspector:
+	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/sync_diff_inspector ./sync_diff_inspector
+
 dm: dm-master dm-worker dmctl dm-syncer
 
 dm-master:
@@ -443,7 +446,7 @@ dm_unit_test_in_verify_ci: check_failpoint_ctl tools/bin/gotestsum tools/bin/goc
 	tools/bin/gocov convert "$(DM_TEST_DIR)/cov.unit_test.out" | tools/bin/gocov-xml > dm-coverage.xml
 	$(FAILPOINT_DISABLE)
 
-dm_integration_test_build: check_failpoint_ctl
+dm_integration_test_build: check_failpoint_ctl sync-diff-inspector
 	$(FAILPOINT_ENABLE)
 	$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
 		-coverpkg=github.com/pingcap/tiflow/dm/... \
@@ -495,9 +498,8 @@ install_test_python_dep:
 	@echo "install python requirments for test"
 	pip install --user -q -r ./dm/tests/requirements.txt
 
-check_third_party_binary_for_dm:
+check_third_party_binary_for_dm : sync-diff-inspector
 	@which bin/tidb-server
-	@which bin/sync_diff_inspector
 	@which mysql
 	@which bin/minio
 
@@ -554,7 +556,7 @@ tiflow-chaos-case:
 engine_unit_test: check_failpoint_ctl
 	$(call run_engine_unit_test,$(ENGINE_PACKAGES))
 
-engine_integration_test: check_third_party_binary_for_engine
+engine_integration_test: check_third_party_binary_for_engine sync-diff-inspector
 	mkdir -p /tmp/tiflow_engine_test || true
 	./engine/test/integration_tests/run.sh "$(CASE)" "$(START_AT)" 2>&1 | tee /tmp/tiflow_engine_test/engine_it.log
 	./engine/test/utils/check_log.sh
@@ -566,7 +568,6 @@ check_third_party_binary_for_engine:
 	@which mysql || (echo "mysql not found in ${PATH}"; exit 1)
 	@which jq || (echo "jq not found in ${PATH}"; exit 1)
 	@which mc || (echo "mc not found in ${PATH}, you can use 'make bin/mc' and move bin/mc to ${PATH}"; exit 1)
-	@which bin/sync_diff_inspector || (echo "run 'make bin/sync_diff_inspector' to download it if you need")
 
 check_engine_integration_test:
 	./engine/test/utils/check_case.sh
@@ -581,9 +582,6 @@ check_cdc_integration_test:
 bin/mc:
 	./scripts/download-mc.sh
 
-bin/sync_diff_inspector:
-	./scripts/download-sync-diff.sh
-
 define run_engine_unit_test
 	@echo "running unit test for packages:" $(1)
 	mkdir -p $(ENGINE_TEST_DIR)

dm/tests/README.md

@@ -4,7 +4,7 @@
 1. The following executables must be copied or generated or linked into these locations.
 
     * `bin/tidb-server` can be downloaded from [tidb-master-linux-amd64](https://download.pingcap.org/tidb-master-linux-amd64.tar.gz) or installed by [tiup](https://github.com/pingcap/tiup), you can use the command `find ~/.tiup -name tidb-server` to locate `tidb-server` binary file and copy it
-    * `bin/sync_diff_inspector` # can be downloaded from [tidb-enterprise-tools-latest-linux-amd64](http://download.pingcap.org/tidb-enterprise-tools-latest-linux-amd64.tar.gz) or build from [source code](https://github.com/pingcap/tidb-tools)
+    * `bin/sync_diff_inspector` # generated by `make dm_integration_test_build`
     * `bin/minio` can be build from (https://github.com/minio/minio)
     * `bin/dm-master.test` # generated by `make dm_integration_test_build`
     * `bin/dm-worker.test` # generated by `make dm_integration_test_build`

dm/tests/download-compatibility-test-binaries.sh

@@ -77,7 +77,6 @@ main() {
 	# Define download URLs
 	local download_urls=(
 		"${FILE_SERVER_URL}/download/builds/pingcap/tidb/${tidb_sha1}/centos7/tidb-server.tar.gz"
-		"http://download.pingcap.org/tidb-enterprise-tools-nightly-linux-amd64.tar.gz"
 		"http://download.pingcap.org/tidb-enterprise-tools-latest-linux-amd64.tar.gz"
 		"${GITHUB_RELEASE_URL}/gh-ost-binary-linux-20200828140552.tar.gz"
 		"${FILE_SERVER_URL}/download/minio.tar.gz"
@@ -98,11 +97,6 @@ main() {
 			extract "$filename" "$THIRD_BIN_DIR" "bin/tidb-server"
 			mv "${THIRD_BIN_DIR}/bin/tidb-server" "$THIRD_BIN_DIR/"
 			;;
-		tidb-enterprise-tools-nightly-linux-amd64.tar.gz)
-			extract "$filename" "$THIRD_BIN_DIR" "tidb-enterprise-tools-nightly-linux-amd64/bin/sync_diff_inspector"
-			mv "${THIRD_BIN_DIR}/tidb-enterprise-tools-nightly-linux-amd64/bin/sync_diff_inspector" "$THIRD_BIN_DIR/"
-			rm -rf "${THIRD_BIN_DIR}/tidb-enterprise-tools-nightly-linux-amd64"
-			;;
 		tidb-enterprise-tools-latest-linux-amd64.tar.gz)
 			extract "$filename" "$THIRD_BIN_DIR" "tidb-enterprise-tools-latest-linux-amd64/bin/mydumper"
 			mv "${THIRD_BIN_DIR}/tidb-enterprise-tools-latest-linux-amd64/bin/mydumper" "$THIRD_BIN_DIR/"

dm/tests/download-integration-test-binaries.sh

@@ -70,14 +70,12 @@ main() {
 	local tidb_sha1=$(get_sha1 "tidb" "$tidb_branch")
 	local tikv_sha1=$(get_sha1 "tikv" "$tikv_branch")
 	local pd_sha1=$(get_sha1 "pd" "$pd_branch")
-	local tidb_tools_sha1=$(curl "${FILE_SERVER_URL}/download/refs/pingcap/tidb-tools/master/sha1")
 
 	# Define download URLs
 	local download_urls=(
 		"${FILE_SERVER_URL}/download/builds/pingcap/tidb/${tidb_sha1}/centos7/tidb-server.tar.gz"
 		"${FILE_SERVER_URL}/download/builds/pingcap/tikv/${tikv_sha1}/centos7/tikv-server.tar.gz"
 		"${FILE_SERVER_URL}/download/builds/pingcap/pd/${pd_sha1}/centos7/pd-server.tar.gz"
-		"${FILE_SERVER_URL}/download/builds/pingcap/tidb-tools/${tidb_tools_sha1}/centos7/tidb-tools.tar.gz"
 		"${GITHUB_RELEASE_URL}/gh-ost-binary-linux-20200828140552.tar.gz"
 		"${FILE_SERVER_URL}/download/minio.tar.gz"
 	)
@@ -105,10 +103,6 @@ main() {
 			tar -xz -C "$THIRD_BIN_DIR" bin/tikv-server -f "${TEMP_DIR}/${filename}"
 			mv "${THIRD_BIN_DIR}/bin/tikv-server" "$THIRD_BIN_DIR/"
 			;;
-		tidb-tools.tar.gz)
-			tar -xz -C "$THIRD_BIN_DIR" 'bin/sync_diff_inspector' -f "${TEMP_DIR}/${filename}"
-			mv "${THIRD_BIN_DIR}/bin/sync_diff_inspector" "$THIRD_BIN_DIR/"
-			;;
 		minio.tar.gz | gh-ost-binary-linux-20200828140552.tar.gz)
 			tar -xz -C "$THIRD_BIN_DIR" -f "${TEMP_DIR}/${filename}"
 			;;

dm/tests/mariadb_master_down_and_up/case.sh

@@ -107,7 +107,6 @@ function clean_task() {
 function test_master_down_and_up() {
 	cleanup_process
 	clean_data
-	install_sync_diff
 	setup_replica
 	gen_full_data
 	run_dm_components_and_create_source $1

dm/tests/mariadb_master_down_and_up/lib.sh

@@ -27,12 +27,6 @@ function exec_tidb() {
 	echo $2 | mysql -uroot -h127.0.0.1 -P$1
 }
 
-function install_sync_diff() {
-	curl https://download.pingcap.org/tidb-enterprise-tools-nightly-linux-amd64.tar.gz | tar xz
-	mkdir -p bin
-	mv tidb-enterprise-tools-nightly-linux-amd64/bin/sync_diff_inspector bin/
-}
-
 function get_master_status() {
 	arr=$(echo "show master status;" | MYSQL_PWD=123456 mysql -uroot -h127.0.0.1 -P3306 | awk 'NR==2')
 	echo $arr

dm/tests/tiup/lib.sh

@@ -56,12 +56,6 @@ function run_sql_tidb_with_retry() {
 	fi
 }
 
-function install_sync_diff() {
-	curl https://download.pingcap.org/tidb-enterprise-tools-nightly-linux-amd64.tar.gz | tar xz
-	mkdir -p bin
-	mv tidb-enterprise-tools-nightly-linux-amd64/bin/sync_diff_inspector bin/
-}
-
 function exec_full_stage() {
 	# drop previous data
 	exec_sql mysql1 3306 "DROP DATABASE IF EXISTS $DB1;"

dm/tests/tiup/upgrade-from-v1.sh

@@ -122,8 +122,6 @@ function destroy_v2_by_tiup() {
 }
 
 function test() {
-	install_sync_diff
-
 	deploy_v1_by_ansible
 
 	migrate_in_v1

dm/tests/tiup/upgrade-from-v2.sh

@@ -170,8 +170,6 @@ function destroy_v2_by_tiup() {
 }
 
 function test() {
-	install_sync_diff
-
 	deploy_previous_v2
 
 	migrate_in_previous_v2

dm/tests/tiup/upgrade-tidb.sh

@@ -52,8 +52,6 @@ function destroy_v2_by_tiup() {
 
 # run this before upgrade TiDB.
 function before_upgrade() {
-	install_sync_diff
-
 	deploy_dm
 
 	migrate_before_upgrade

dm/tests/upstream_switch/case.sh

@@ -209,7 +209,6 @@ function check_master() {
 function test_relay() {
 	cleanup_process
 	check_master
-	install_sync_diff
 	clean_data
 	prepare_binlogs
 	setup_replica

dm/tests/upstream_switch/lib.sh

@@ -30,12 +30,6 @@ function exec_tidb() {
 	echo $2 | mysql -uroot -h$1 -P4000
 }
 
-function install_sync_diff() {
-	curl https://download.pingcap.org/tidb-enterprise-tools-nightly-linux-amd64.tar.gz | tar xz
-	mkdir -p bin
-	mv tidb-enterprise-tools-nightly-linux-amd64/bin/sync_diff_inspector bin/
-}
-
 function prepare_more_binlogs() {
 	exec_sql $1 "create database db1 collate latin1_bin;"
 	exec_sql $1 "flush logs;"

scripts/download-integration-test-binaries.sh

@@ -91,7 +91,7 @@ download_community_binaries() {
 	mv ${THIRD_BIN_DIR}/tiflash ${THIRD_BIN_DIR}/_tiflash
 	mv ${THIRD_BIN_DIR}/_tiflash/* ${THIRD_BIN_DIR} && rm -rf ${THIRD_BIN_DIR}/_tiflash
 	tar -xz -C ${THIRD_BIN_DIR} pd-ctl -f ${TMP_DIR}/$tidb_file_name/ctl-${dist}.tar.gz
-	tar -xz -C ${THIRD_BIN_DIR} $toolkit_file_name/etcdctl $toolkit_file_name/sync_diff_inspector -f ${TMP_DIR}/$toolkit_tar_name
+	tar -xz -C ${THIRD_BIN_DIR} $toolkit_file_name/etcdctl -f ${TMP_DIR}/$toolkit_tar_name
 	mv ${THIRD_BIN_DIR}/$toolkit_file_name/* ${THIRD_BIN_DIR} && rm -rf ${THIRD_BIN_DIR}/$toolkit_file_name
 
 	# Download additional tools
@@ -147,7 +147,6 @@ download_binaries() {
 	local minio_download_url="${FILE_SERVER_URL}/download/minio.tar.gz"
 	local go_ycsb_download_url="${FILE_SERVER_URL}/download/builds/pingcap/go-ycsb/test-br/go-ycsb"
 	local etcd_download_url="${FILE_SERVER_URL}/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz"
-	local sync_diff_inspector_url="${FILE_SERVER_URL}/download/builds/pingcap/cdc/sync_diff_inspector_hash-a129f096_linux-amd64.tar.gz"
 	local jq_download_url="${FILE_SERVER_URL}/download/builds/pingcap/test/jq-1.6/jq-linux64"
 	local schema_registry_url="${FILE_SERVER_URL}/download/builds/pingcap/cdc/schema-registry.tar.gz"
 
@@ -158,7 +157,6 @@ download_binaries() {
 	download_and_extract "$tiflash_download_url" "tiflash.tar.gz"
 	download_and_extract "$minio_download_url" "minio.tar.gz"
 	download_and_extract "$etcd_download_url" "etcd.tar.gz" "etcd-v3.4.7-linux-amd64/etcdctl"
-	download_and_extract "$sync_diff_inspector_url" "sync_diff_inspector.tar.gz"
 	download_and_extract "$schema_registry_url" "schema-registry.tar.gz"
 
 	download_file "$go_ycsb_download_url" "go-ycsb" "${THIRD_BIN_DIR}/go-ycsb"

sync_diff_inspector/checkpoints/checkpoints.go

@@ -211,7 +211,7 @@ func (cp *Checkpoint) Init() {
 		Nodes: make([]*Node, 0),
 		CurrentSavedNode: &Node{
 			ChunkRange: &chunk.Range{
-				Index:   chunk.GetInitCID(),
+				Index:   chunk.GetInitChunkID(),
 				IsFirst: true,
 				IsLast:  true,
 			},

sync_diff_inspector/chunk/chunk.go

@@ -68,8 +68,8 @@ type CID struct {
 	ChunkCnt int `json:"chunk-count"`
 }
 
-// GetInitCID return an empty CID
-func GetInitCID() *CID {
+// GetInitChunkID return an empty CID
+func GetInitChunkID() *CID {
 	return &CID{
 		TableIndex:       -1,
 		BucketIndexLeft:  -1,

sync_diff_inspector/diff/diff.go

@@ -229,7 +229,7 @@ func (df *Diff) initCheckpoint() error {
 	return nil
 }
 
-func encodeConfig(config *report.Config) ([]byte, error) {
+func encodeReportConfig(config *report.Config) ([]byte, error) {
 	buf := new(bytes.Buffer)
 	if err := toml.NewEncoder(buf).Encode(config); err != nil {
 		return nil, errors.Trace(err)
@@ -261,12 +261,12 @@ func getConfigsForReport(cfg *config.Config) ([][]byte, []byte, error) {
 	sourceBytes := make([][]byte, len(sourceConfigs))
 	var err error
 	for i := range sourceBytes {
-		sourceBytes[i], err = encodeConfig(sourceConfigs[i])
+		sourceBytes[i], err = encodeReportConfig(sourceConfigs[i])
 		if err != nil {
 			return nil, nil, errors.Trace(err)
 		}
 	}
-	targetBytes, err := encodeConfig(targetConfig)
+	targetBytes, err := encodeReportConfig(targetConfig)
 	if err != nil {
 		return nil, nil, errors.Trace(err)
 	}
@@ -863,7 +863,7 @@ func (df *Diff) removeSQLFiles(checkPointID *chunk.CID) error {
 			if len(fileIDSubstrs) != 3 {
 				return nil
 			}
-			tableIndex, bucketIndexLeft, bucketIndexRight, chunkIndex, err := utils.GetCIDFromSQLFileName(fileIDSubstrs[2])
+			tableIndex, bucketIndexLeft, bucketIndexRight, chunkIndex, err := utils.GetChunkIDFromSQLFileName(fileIDSubstrs[2])
 			if err != nil {
 				return errors.Trace(err)
 			}