From 856e4f9dfe11bb804d782243878babde91347b32 Mon Sep 17 00:00:00 2001 From: Grace Cai Date: Tue, 16 Apr 2024 11:54:06 +0800 Subject: [PATCH] ticdc: refine changefeed related documentations (#16963) --- releases/release-7.5.0.md | 1 + ticdc/ticdc-changefeed-config.md | 6 ++++++ ticdc/ticdc-changefeed-overview.md | 2 +- ticdc/ticdc-ddl.md | 4 ++++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/releases/release-7.5.0.md b/releases/release-7.5.0.md index 085a1ed5cd9a4..28f93759a0d75 100644 --- a/releases/release-7.5.0.md +++ b/releases/release-7.5.0.md @@ -180,6 +180,7 @@ Compared with the previous LTS 7.1.0, 7.5.0 includes new features, improvements, | BR | [`--ignore-stats`](/br/br-snapshot-manual.md#back-up-statistics) | Newly added | Controls whether to back up and restore database statistics. When you set this parameter to `false`, the br command-line tool supports backing up and restoring statistics of columns, indexes, and tables. | | TiCDC | [`case-sensitive`](/ticdc/ticdc-changefeed-config.md) | Modified | Changes the default value from `true` to `false` after further tests, which means that the table names and database names in the TiCDC configuration file are case-insensitive by default. | | TiCDC | [`sink.dispatchers.partition`](/ticdc/ticdc-changefeed-config.md) | Modified | Controls how TiCDC dispatches incremental data to Kafka partitions. v7.5.0 introduces a new value option `columns`, which uses the explicitly specified column values to calculate the partition number. | +| TiCDC | [`changefeed-error-stuck-duration`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the duration for which the changefeed is allowed to automatically retry when internal errors or exceptions occur. | | TiCDC | [`encoding-worker-num`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the number of encoding and decoding workers in the redo module. | | TiCDC | [`flush-worker-num`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the number of flushing workers in the redo module. | | TiCDC | [`sink.column-selectors`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the specified columns of data change events that TiCDC sends to Kafka when dispatching incremental data. | diff --git a/ticdc/ticdc-changefeed-config.md b/ticdc/ticdc-changefeed-config.md index 3273c7e3875f9..6dc5eb1482f35 100644 --- a/ticdc/ticdc-changefeed-config.md +++ b/ticdc/ticdc-changefeed-config.md @@ -68,6 +68,12 @@ case-sensitive = false # The default value is the same as the default SQL mode of TiDB. # sql-mode = "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" +# The duration for which the changefeed is allowed to automatically retry when internal errors or exceptions occur. The default value is 30 minutes. +# The changefeed enters the failed state if internal errors or exceptions occur in the changefeed and persist longer than the duration set by this parameter. +# When the changefeed is in the failed state, you need to restart the changefeed manually for recovery. +# The format of this parameter is "h m s", for example, "1h30m30s". +changefeed-error-stuck-duration = "30m" + [mounter] # The number of threads with which the mounter decodes KV data. The default value is 16. # worker-num = 16 diff --git a/ticdc/ticdc-changefeed-overview.md b/ticdc/ticdc-changefeed-overview.md index 30517454d1a32..0c63f6f60f357 100644 --- a/ticdc/ticdc-changefeed-overview.md +++ b/ticdc/ticdc-changefeed-overview.md @@ -17,7 +17,7 @@ The states in the preceding state transfer diagram are described as follows: - `Normal`: The replication task runs normally and the checkpoint-ts proceeds normally. A changefeed in this state blocks GC to advance. - `Stopped`: The replication task is stopped, because the user manually pauses the changefeed. The changefeed in this state blocks GC operations. -- `Warning`: The replication task returns an error. The replication cannot continue due to some recoverable errors. The changefeed in this state keeps trying to resume until the state transfers to `Normal`. The maximum retry time is 30 minutes. If it exceeds this time, the changefeed enters a failed state. The changefeed in this state blocks GC operations. +- `Warning`: The replication task returns an error. The replication cannot continue due to some recoverable errors. The changefeed in this state keeps trying to resume until the state transfers to `Normal`. The default retry time is 30 minutes (which can be adjusted by [`changefeed-error-stuck-duration`](/ticdc/ticdc-changefeed-config.md)). If it exceeds this time, the changefeed enters a failed state. The changefeed in this state blocks GC operations. - `Finished`: The replication task is finished and has reached the preset `TargetTs`. The changefeed in this state does not block GC operations. - `Failed`: The replication task fails. The changefeed in this state does not keep trying to resume. To give you enough time to handle the failure, the changefeed in this state blocks GC operations. The duration of the blockage is specified by the `gc-ttl` parameter, with a default value of 24 hours. If the underlying issue is resolved within this duration, you can manually resume the changefeed. Otherwise, if the changefeed remains in this state beyond the `gc-ttl` duration, the replication task cannot resume and cannot be recovered. diff --git a/ticdc/ticdc-ddl.md b/ticdc/ticdc-ddl.md index 1d05faf4d2c58..6e6dbbd3ae470 100644 --- a/ticdc/ticdc-ddl.md +++ b/ticdc/ticdc-ddl.md @@ -102,6 +102,10 @@ TiCDC processes this type of DDL as follows: | `RENAME TABLE test.t1 TO ignore.t1, test.t2 TO test.t22;` | Report an error | The new database name `ignore` does not match the filter rule. | | `RENAME TABLE test.t1 TO test.t4, test.t3 TO test.t1, test.t4 TO test.t3;` | Report an error | The `RENAME TABLE` DDL swaps the names of `test.t1` and `test.t3` in one DDL statement, which TiCDC cannot handle correctly. In this case, refer to the error message for handling. | +### DDL statement considerations + +When executing cross-database DDL statements (such as `CREATE TABLE db1.t1 LIKE t2`) in the upstream, it is recommended that you explicitly specify all relevant database names in DDL statements (such as `CREATE TABLE db1.t1 LIKE db2.t2`). Otherwise, cross-database DDL statements might not be executed correctly in the downstream due to the lack of database name information. + ### SQL mode By default, TiCDC uses the default SQL mode of TiDB to parse DDL statements. If your upstream TiDB cluster uses a non-default SQL mode, you must specify the SQL mode in the TiCDC configuration file. Otherwise, TiCDC might fail to parse DDL statements correctly. For more information about TiDB SQL mode, see [SQL Mode](/sql-mode.md).