test(engine, dm): add task resume integration test (#7631)

ref #6776
pingcap · Nov 24, 2022 · 8346e97 · 8346e97
1 parent 573d752
commit 8346e97
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 7 deletions.
diff --git a/deployments/engine/docker-compose/dm_databases.yaml b/deployments/engine/docker-compose/dm_databases.yaml
@@ -22,6 +22,7 @@ services:
 
     dm_downstream_pd:
         image: pingcap/pd:nightly
+        container_name: dm_downstream_pd
         ports:
             - "2379"
         command:
@@ -36,6 +37,7 @@ services:
 
     dm_downstream_tikv:
         image: pingcap/tikv:nightly
+        container_name: dm_downstream_tikv
         command:
             - --addr=0.0.0.0:20160
             - --advertise-addr=dm_downstream_tikv:20160
@@ -46,6 +48,7 @@ services:
 
     dm_downstream_tidb:
         image: pingcap/tidb:nightly
+        container_name: dm_downstream_tidb
         ports:
             - "4000:4000"
             - "10080:10080"

diff --git a/deployments/engine/docker-compose/dm_databases_tidb_new_collation_off.yaml b/deployments/engine/docker-compose/dm_databases_tidb_new_collation_off.yaml
@@ -22,6 +22,7 @@ services:
 
     dm_downstream_pd:
         image: pingcap/pd:nightly
+        container_name: dm_downstream_pd
         ports:
             - "2379"
         command:
@@ -36,6 +37,7 @@ services:
 
     dm_downstream_tikv:
         image: pingcap/tikv:nightly
+        container_name: dm_downstream_tikv
         command:
             - --addr=0.0.0.0:20160
             - --advertise-addr=dm_downstream_tikv:20160
@@ -46,6 +48,7 @@ services:
 
     dm_downstream_tidb:
         image: pingcap/tidb:nightly
+        container_name: dm_downstream_tidb
         ports:
             - "4000:4000"
             - "10080:10080"

diff --git a/deployments/engine/docker-compose/dm_databases_with_tls.yaml b/deployments/engine/docker-compose/dm_databases_with_tls.yaml
@@ -26,6 +26,7 @@ services:
 
     dm_downstream_pd:
         image: pingcap/pd:nightly
+        container_name: dm_downstream_pd
         ports:
             - "2379"
         command:
@@ -40,6 +41,7 @@ services:
 
     dm_downstream_tikv:
         image: pingcap/tikv:nightly
+        container_name: dm_downstream_tikv
         command:
             - --addr=0.0.0.0:20160
             - --advertise-addr=dm_downstream_tikv:20160
@@ -50,6 +52,7 @@ services:
 
     dm_downstream_tidb:
         image: pingcap/tidb:nightly
+        container_name: dm_downstream_tidb
         ports:
             - "4000:4000"
             - "10080:10080"

diff --git a/engine/test/integration_tests/dm_many_tables/run.sh b/engine/test/integration_tests/dm_many_tables/run.sh
@@ -27,16 +27,41 @@ function run() {
 		run_sql --quiet "INSERT INTO dm_many_tables.t$i VALUES (9, 90009);"
 	done
 
-	# create job & wait for job finished
+	# create job & wait for job to enter load phase
 	job_id=$(create_job "DM" "$CUR_DIR/conf/job.yaml" "dm_many_tables")
-	# check progress is forwarded gradually, not jump to "finished"
 	exec_with_retry --count 500 "curl \"http://127.0.0.1:10245/api/v1/jobs/$job_id/status\" | tee /dev/stderr | jq -e '.task_status.\"mysql-01\".status.status | .finishedBytes > 0 and .finishedBytes < .totalBytes'"
-	exec_with_retry --count 100 "curl \"http://127.0.0.1:10245/api/v1/jobs/$job_id\" | tee /dev/stderr | jq -e '.state == \"Finished\"'"
 
-	# check data
+	# test autoresume
+	docker stop dm_downstream_tidb
+	exec_with_retry --count 20 "curl \"http://127.0.0.1:10245/api/v1/jobs/$job_id/status\" | tee /dev/stderr | jq -e '.task_status.\"mysql-01\".status | .unit == \"DMLoadTask\" and .stage == \"Error\"'"
+	docker start dm_downstream_tidb
+	docker restart server-executor-0 server-executor-1 server-executor-2
+
+	# wait jobMaster online
+	exec_with_retry --count 50 --interval_sec 10 "curl \"http://127.0.0.1:10245/api/v1/jobs/$job_id/status\" | grep 'job_id'"
+	exec_with_retry --count 500 "curl \"http://127.0.0.1:10245/api/v1/jobs/$job_id/status\" | tee /dev/stderr | jq -e '.task_status.\"mysql-01\".status.status | .finishedBytes > 0 and .finishedBytes < .totalBytes'"
+
+	# test pause and resume
+	exec_with_retry --count 20 "curl -X PUT \"http://127.0.0.1:10245/api/v1/jobs/$job_id/status\" -H 'Content-Type: application/json' -d '{\"op\": \"pause\"}'"
+	sleep 10
+	exec_with_retry --count 20 "curl \"http://127.0.0.1:10245/api/v1/jobs/$job_id/status\" | tee /dev/stderr | jq -e '.task_status.\"mysql-01\".status | .stage == \"Paused\"'"
+	exec_with_retry --count 20 "curl -X PUT \"http://127.0.0.1:10245/api/v1/jobs/$job_id/status\" -H 'Content-Type: application/json' -d '{\"op\": \"resume\"}'"
+	exec_with_retry --count 20 "curl \"http://127.0.0.1:10245/api/v1/jobs/$job_id/status\" | tee /dev/stderr | jq -e '.task_status.\"mysql-01\".status | .stage == \"Running\"'"
+
+	# wait for job finished and check data
+	exec_with_retry --count 50 --interval_sec 10 "curl \"http://127.0.0.1:10245/api/v1/jobs/$job_id\" | tee /dev/stderr | jq -e '.state == \"Finished\"'"
 	check_sync_diff $WORK_DIR $CUR_DIR/conf/diff_config.toml 1
 }
 
-trap "stop_engine_cluster $WORK_DIR $CONFIG" EXIT
+function stop {
+	if [ ! -z $job_id ]; then
+		echo -e "\n\nquery job statu before stop dm_many_tables...\n"
+		curl "http://127.0.0.1:10245/api/v1/jobs/$job_id/status" || true
+		curl "http://127.0.0.1:10245/api/v1/jobs/$job_id" || true
+	fi
+	stop_engine_cluster $WORK_DIR $CONFIG
+}
+
+trap stop EXIT
 run $*
 echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"
diff --git a/engine/test/utils/check_sync_diff b/engine/test/utils/check_sync_diff
@@ -12,8 +12,8 @@ else
 	check_time=20
 fi
 
-PWD=$(pwd)
-binary=$PWD/bin/sync_diff_inspector
+PWD=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+binary=$PWD/../../../bin/sync_diff_inspector
 OUTPUT_DIR=/tmp/engine_test/output/
 STDOUT=$workdir/sync_diff_stdout.log