From b16d887dfefc7619b6388635918c36560ae68b1c Mon Sep 17 00:00:00 2001 From: Kathryn May <44557882+kathancox@users.noreply.github.com> Date: Tue, 6 Aug 2024 13:04:44 -0400 Subject: [PATCH] Add `schema_locked` storage parameter for changefeed performance improvement (#18763) --- .../v23.1/misc/table-storage-parameters.md | 1 + .../v23.2/misc/table-storage-parameters.md | 1 + .../v24.1/misc/table-storage-parameters.md | 1 + .../v24.2/misc/table-storage-parameters.md | 1 + src/current/v23.1/create-changefeed.md | 18 ++++++++++++++++++ src/current/v23.2/create-changefeed.md | 18 ++++++++++++++++++ src/current/v24.1/create-changefeed.md | 18 ++++++++++++++++++ src/current/v24.2/create-changefeed.md | 18 ++++++++++++++++++ 8 files changed, 76 insertions(+) diff --git a/src/current/_includes/v23.1/misc/table-storage-parameters.md b/src/current/_includes/v23.1/misc/table-storage-parameters.md index 2f0f43b57b5..0efd01751be 100644 --- a/src/current/_includes/v23.1/misc/table-storage-parameters.md +++ b/src/current/_includes/v23.1/misc/table-storage-parameters.md @@ -1,6 +1,7 @@ | Parameter name | Description | Data type | Default value | |------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|-----------|---------------| | `exclude_data_from_backup` | Exclude the data in this table from any future backups. | Boolean | `false` | +| New in v23.1.15: `schema_locked` | Disallow [schema changes]({% link {{ page.version.version }}/online-schema-changes.md %}) on this table. Enabling `schema_locked` can help [improve performance of changefeeds]({% link {{ page.version.version }}/create-changefeed.md %}#disallow-schema-changes-on-tables-to-improve-changefeed-performance) running on this table. | Boolean | `false` | | `sql_stats_automatic_collection_enabled` | Enable [automatic statistics collection]({% link {{ page.version.version }}/cost-based-optimizer.md %}#enable-and-disable-automatic-statistics-collection-for-tables) for this table. | Boolean | `true` | | `sql_stats_automatic_collection_min_stale_rows` | Minimum number of stale rows in this table that will trigger a statistics refresh. | Integer | 500 | | `sql_stats_automatic_collection_fraction_stale_rows` | Fraction of stale rows in this table that will trigger a statistics refresh. | Float | 0.2 | diff --git a/src/current/_includes/v23.2/misc/table-storage-parameters.md b/src/current/_includes/v23.2/misc/table-storage-parameters.md index 2f0f43b57b5..32dee5a9aaa 100644 --- a/src/current/_includes/v23.2/misc/table-storage-parameters.md +++ b/src/current/_includes/v23.2/misc/table-storage-parameters.md @@ -1,6 +1,7 @@ | Parameter name | Description | Data type | Default value | |------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|-----------|---------------| | `exclude_data_from_backup` | Exclude the data in this table from any future backups. | Boolean | `false` | +| New in v23.2.1: `schema_locked` | Disallow [schema changes]({% link {{ page.version.version }}/online-schema-changes.md %}) on this table. Enabling `schema_locked` can help [improve performance of changefeeds]({% link {{ page.version.version }}/create-changefeed.md %}#disallow-schema-changes-on-tables-to-improve-changefeed-performance) running on this table. | Boolean | `false` | | `sql_stats_automatic_collection_enabled` | Enable [automatic statistics collection]({% link {{ page.version.version }}/cost-based-optimizer.md %}#enable-and-disable-automatic-statistics-collection-for-tables) for this table. | Boolean | `true` | | `sql_stats_automatic_collection_min_stale_rows` | Minimum number of stale rows in this table that will trigger a statistics refresh. | Integer | 500 | | `sql_stats_automatic_collection_fraction_stale_rows` | Fraction of stale rows in this table that will trigger a statistics refresh. | Float | 0.2 | diff --git a/src/current/_includes/v24.1/misc/table-storage-parameters.md b/src/current/_includes/v24.1/misc/table-storage-parameters.md index 2f0f43b57b5..3ca7f601648 100644 --- a/src/current/_includes/v24.1/misc/table-storage-parameters.md +++ b/src/current/_includes/v24.1/misc/table-storage-parameters.md @@ -1,6 +1,7 @@ | Parameter name | Description | Data type | Default value | |------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|-----------|---------------| | `exclude_data_from_backup` | Exclude the data in this table from any future backups. | Boolean | `false` | +| `schema_locked` | Disallow [schema changes]({% link {{ page.version.version }}/online-schema-changes.md %}) on this table. Enabling `schema_locked` can help [improve performance of changefeeds]({% link {{ page.version.version }}/create-changefeed.md %}#disallow-schema-changes-on-tables-to-improve-changefeed-performance) running on this table. | Boolean | `false` | | `sql_stats_automatic_collection_enabled` | Enable [automatic statistics collection]({% link {{ page.version.version }}/cost-based-optimizer.md %}#enable-and-disable-automatic-statistics-collection-for-tables) for this table. | Boolean | `true` | | `sql_stats_automatic_collection_min_stale_rows` | Minimum number of stale rows in this table that will trigger a statistics refresh. | Integer | 500 | | `sql_stats_automatic_collection_fraction_stale_rows` | Fraction of stale rows in this table that will trigger a statistics refresh. | Float | 0.2 | diff --git a/src/current/_includes/v24.2/misc/table-storage-parameters.md b/src/current/_includes/v24.2/misc/table-storage-parameters.md index 2f0f43b57b5..3ca7f601648 100644 --- a/src/current/_includes/v24.2/misc/table-storage-parameters.md +++ b/src/current/_includes/v24.2/misc/table-storage-parameters.md @@ -1,6 +1,7 @@ | Parameter name | Description | Data type | Default value | |------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|-----------|---------------| | `exclude_data_from_backup` | Exclude the data in this table from any future backups. | Boolean | `false` | +| `schema_locked` | Disallow [schema changes]({% link {{ page.version.version }}/online-schema-changes.md %}) on this table. Enabling `schema_locked` can help [improve performance of changefeeds]({% link {{ page.version.version }}/create-changefeed.md %}#disallow-schema-changes-on-tables-to-improve-changefeed-performance) running on this table. | Boolean | `false` | | `sql_stats_automatic_collection_enabled` | Enable [automatic statistics collection]({% link {{ page.version.version }}/cost-based-optimizer.md %}#enable-and-disable-automatic-statistics-collection-for-tables) for this table. | Boolean | `true` | | `sql_stats_automatic_collection_min_stale_rows` | Minimum number of stale rows in this table that will trigger a statistics refresh. | Integer | 500 | | `sql_stats_automatic_collection_fraction_stale_rows` | Fraction of stale rows in this table that will trigger a statistics refresh. | Float | 0.2 | diff --git a/src/current/v23.1/create-changefeed.md b/src/current/v23.1/create-changefeed.md index b345e05665d..3d586b39ca7 100644 --- a/src/current/v23.1/create-changefeed.md +++ b/src/current/v23.1/create-changefeed.md @@ -335,6 +335,24 @@ CREATE CHANGEFEED FOR TABLE table_name INTO 'external://kafka_sink' WITH resolved; ~~~ +### Disallow schema changes on tables to improve changefeed performance + +{% include_cached new-in.html version="v23.1.15" %} Use the `schema_locked` [storage parameter]({% link {{ page.version.version }}/with-storage-parameter.md %}) to disallow [schema changes]({% link {{ page.version.version }}/online-schema-changes.md %}) on a watched table, which helps to decrease the latency between a write committing to a table and it emitting to the [changefeed's sink]({% link {{ page.version.version }}/changefeed-sinks.md %}). You can lock the table before creating a changefeed or while a changefeed is running, which will enable the performance improvement for changefeeds watching the particular table. + +Enable `schema_locked` on the watched table with the [`ALTER TABLE`]({% link {{ page.version.version }}/alter-table.md %}) statement: + +{% include_cached copy-clipboard.html %} +~~~ sql +ALTER TABLE watched_table SET (schema_locked = true); +~~~ + +While `schema_locked` is enabled on a table, attempted schema changes on the table will be rejected and an error returned. If you need to run a schema change on the locked table, unlock the table with `schema_locked = false`, complete the schema change, and then lock the table again with `schema_locked = true`. The changefeed will run as normal while `schema_locked = false`, but it will not benefit from the performance optimization. + +{% include_cached copy-clipboard.html %} +~~~ sql +ALTER TABLE watched_table SET (schema_locked = false); +~~~ + ### Manage a changefeed For {{ site.data.products.enterprise }} changefeeds, use [`SHOW CHANGEFEED JOBS`]({% link {{ page.version.version }}/show-jobs.md %}) to check the status of your changefeed jobs: diff --git a/src/current/v23.2/create-changefeed.md b/src/current/v23.2/create-changefeed.md index 60dd556a568..03388edafc3 100644 --- a/src/current/v23.2/create-changefeed.md +++ b/src/current/v23.2/create-changefeed.md @@ -341,6 +341,24 @@ CREATE CHANGEFEED FOR TABLE table_name INTO 'external://kafka_sink' WITH resolved; ~~~ +### Disallow schema changes on tables to improve changefeed performance + +{% include_cached new-in.html version="v23.2.1" %} Use the `schema_locked` [storage parameter]({% link {{ page.version.version }}/with-storage-parameter.md %}) to disallow [schema changes]({% link {{ page.version.version }}/online-schema-changes.md %}) on a watched table, which helps to decrease the latency between a write committing to a table and it emitting to the [changefeed's sink]({% link {{ page.version.version }}/changefeed-sinks.md %}). You can lock the table before creating a changefeed or while a changefeed is running, which will enable the performance improvement for changefeeds watching the particular table. + +Enable `schema_locked` on the watched table with the [`ALTER TABLE`]({% link {{ page.version.version }}/alter-table.md %}) statement: + +{% include_cached copy-clipboard.html %} +~~~ sql +ALTER TABLE watched_table SET (schema_locked = true); +~~~ + +While `schema_locked` is enabled on a table, attempted schema changes on the table will be rejected and an error returned. If you need to run a schema change on the locked table, unlock the table with `schema_locked = false`, complete the schema change, and then lock the table again with `schema_locked = true`. The changefeed will run as normal while `schema_locked = false`, but it will not benefit from the performance optimization. + +{% include_cached copy-clipboard.html %} +~~~ sql +ALTER TABLE watched_table SET (schema_locked = false); +~~~ + ### Manage a changefeed For {{ site.data.products.enterprise }} changefeeds, use [`SHOW CHANGEFEED JOBS`]({% link {{ page.version.version }}/show-jobs.md %}) to check the status of your changefeed jobs: diff --git a/src/current/v24.1/create-changefeed.md b/src/current/v24.1/create-changefeed.md index 41df691fe90..9da4cbae89b 100644 --- a/src/current/v24.1/create-changefeed.md +++ b/src/current/v24.1/create-changefeed.md @@ -365,6 +365,24 @@ CREATE CHANGEFEED FOR TABLE table_name INTO 'external://kafka_sink' For guidance on how to filter changefeed messages to emit [row-level TTL]({% link {{ page.version.version }}/row-level-ttl.md %}) deletes only, refer to [Change Data Capture Queries]({% link {{ page.version.version }}/cdc-queries.md %}#reference-ttl-in-a-cdc-query). +### Disallow schema changes on tables to improve changefeed performance + +Use the `schema_locked` [storage parameter]({% link {{ page.version.version }}/with-storage-parameter.md %}) to disallow [schema changes]({% link {{ page.version.version }}/online-schema-changes.md %}) on a watched table, which helps to decrease the latency between a write committing to a table and it emitting to the [changefeed's sink]({% link {{ page.version.version }}/changefeed-sinks.md %}). You can lock the table before creating a changefeed or while a changefeed is running, which will enable the performance improvement for changefeeds watching the particular table. + +Enable `schema_locked` on the watched table with the [`ALTER TABLE`]({% link {{ page.version.version }}/alter-table.md %}) statement: + +{% include_cached copy-clipboard.html %} +~~~ sql +ALTER TABLE watched_table SET (schema_locked = true); +~~~ + +While `schema_locked` is enabled on a table, attempted schema changes on the table will be rejected and an error returned. If you need to run a schema change on the locked table, unlock the table with `schema_locked = false`, complete the schema change, and then lock the table again with `schema_locked = true`. The changefeed will run as normal while `schema_locked = false`, but it will not benefit from the performance optimization. + +{% include_cached copy-clipboard.html %} +~~~ sql +ALTER TABLE watched_table SET (schema_locked = false); +~~~ + ### Manage a changefeed For {{ site.data.products.enterprise }} changefeeds, use [`SHOW CHANGEFEED JOBS`]({% link {{ page.version.version }}/show-jobs.md %}) to check the status of your changefeed jobs: diff --git a/src/current/v24.2/create-changefeed.md b/src/current/v24.2/create-changefeed.md index 8700bc37ed2..5789cb4046f 100644 --- a/src/current/v24.2/create-changefeed.md +++ b/src/current/v24.2/create-changefeed.md @@ -366,6 +366,24 @@ CREATE CHANGEFEED FOR TABLE table_name INTO 'external://kafka_sink' For guidance on how to filter changefeed messages to emit [row-level TTL]({% link {{ page.version.version }}/row-level-ttl.md %}) deletes only, refer to [Change Data Capture Queries]({% link {{ page.version.version }}/cdc-queries.md %}#reference-ttl-in-a-cdc-query). +### Disallow schema changes on tables to improve changefeed performance + +Use the `schema_locked` [storage parameter]({% link {{ page.version.version }}/with-storage-parameter.md %}) to disallow [schema changes]({% link {{ page.version.version }}/online-schema-changes.md %}) on a watched table, which helps to decrease the latency between a write committing to a table and it emitting to the [changefeed's sink]({% link {{ page.version.version }}/changefeed-sinks.md %}). You can lock the table before creating a changefeed or while a changefeed is running, which will enable the performance improvement for changefeeds watching the particular table. + +Enable `schema_locked` on the watched table with the [`ALTER TABLE`]({% link {{ page.version.version }}/alter-table.md %}) statement: + +{% include_cached copy-clipboard.html %} +~~~ sql +ALTER TABLE watched_table SET (schema_locked = true); +~~~ + +While `schema_locked` is enabled on a table, attempted schema changes on the table will be rejected and an error returned. If you need to run a schema change on the locked table, unlock the table with `schema_locked = false`, complete the schema change, and then lock the table again with `schema_locked = true`. The changefeed will run as normal while `schema_locked = false`, but it will not benefit from the performance optimization. + +{% include_cached copy-clipboard.html %} +~~~ sql +ALTER TABLE watched_table SET (schema_locked = false); +~~~ + ### Manage a changefeed For {{ site.data.products.enterprise }} changefeeds, use [`SHOW CHANGEFEED JOBS`]({% link {{ page.version.version }}/show-jobs.md %}) to check the status of your changefeed jobs: