From b5a9d74222d396dcf6af883007df1858c5ded10a Mon Sep 17 00:00:00 2001 From: Charlotte Liu <37295236+CharLotteiu@users.noreply.github.com> Date: Thu, 24 Dec 2020 18:06:38 +0800 Subject: [PATCH 1/7] update GC for release 5.0 --- garbage-collection-configuration.md | 50 +++++++++++++++++++++++++++-- tikv-configuration-file.md | 8 +++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/garbage-collection-configuration.md b/garbage-collection-configuration.md index 842c06ba2c5e8..b973182ef119f 100644 --- a/garbage-collection-configuration.md +++ b/garbage-collection-configuration.md @@ -72,9 +72,9 @@ update mysql.tidb set VARIABLE_VALUE="24h" where VARIABLE_NAME="tikv_gc_life_tim - Specifies the GC mode. Possible values are: - - `"distributed"` (default): Distributed GC mode. In the [Do GC](/garbage-collection-overview.md#do-gc) step, the GC leader on the TiDB side uploads the safe point to PD. Each TiKV node obtains the safe point respectively and performs GC on all leader Regions on the current node. This mode is is supported from TiDB 3.0. + - `"distributed"` (default): Distributed GC mode. In the [Do GC](/garbage-collection-overview.md#do-gc) step, the GC leader on the TiDB side uploads the safe point to PD. Each TiKV node obtains the safe point respectively and performs GC on all leader Regions on the current node. This mode is supported from TiDB 3.0. - - `"central"`: Central GC mode. In the [Do GC](/garbage-collection-overview.md#do-gc) step, the GC leader sends GC requests to all Regions. This mode is adopted by TiDB 2.1 or earlier versions. + - `"central"`: Central GC mode. In the [Do GC](/garbage-collection-overview.md#do-gc) step, the GC leader sends GC requests to all Regions. This mode is adopted by TiDB 2.1 or earlier versions. Starting from TiKV 5.0, this mode is not supported. Clusters set to this mode automatically switch to the `distributed` mode. ## `tikv_gc_auto_concurrency` @@ -143,3 +143,49 @@ You can dynamically modify this configuration using tikv-ctl: ```bash tikv-ctl --host=ip:port modify-tikv-config -m server -n gc.max_write_bytes_per_sec -v 10MB ``` + +## GC in Compaction Filter + +From v5.0, TiKV introduces the mechanism of GC in Compaction Filter. Based on the distributed GC mode, the mechanism uses the compaction process of RocksDB, instead of a separate GC worker thread, to run GC. It helps to avoid extra disk read caused by GC. Also, after clearing the obsolete data, it prevents leaving behind a large number of deletion marks which degrade the sequential scan performance. This mechanism is enabled by default and is also automatically enabled after the rolling upgrade is completed. The following example shows how to enable this GC mechanism in the TiKV configuration file: + +{{< copyable "" >}} + +```toml +[gc] +enable-compaction-filter = true +``` + +You can also enable this GC mechanism by modifying the configuration online. See the following example: + +{{< copyable "sql" >}} + +```sql +show config where type = 'tikv' and name like '%enable-compaction-filter%'; +``` + +```sql ++------+-------------------+-----------------------------+-------+ +| Type | Instance | Name | Value | ++------+-------------------+-----------------------------+-------+ +| tikv | 172.16.5.37:20163 | gc.enable-compaction-filter | false | +| tikv | 172.16.5.36:20163 | gc.enable-compaction-filter | false | +| tikv | 172.16.5.35:20163 | gc.enable-compaction-filter | false | ++------+-------------------+-----------------------------+-------+ +``` + +{{< copyable "sql" >}} + +```sql +set config tikv gc.enable-compaction-filter = true; +show config where type = 'tikv' and name like '%enable-compaction-filter%'; +``` + +```sql ++------+-------------------+-----------------------------+-------+ +| Type | Instance | Name | Value | ++------+-------------------+-----------------------------+-------+ +| tikv | 172.16.5.37:20163 | gc.enable-compaction-filter | true | +| tikv | 172.16.5.36:20163 | gc.enable-compaction-filter | true | +| tikv | 172.16.5.35:20163 | gc.enable-compaction-filter | true | ++------+-------------------+-----------------------------+-------+ +``` \ No newline at end of file diff --git a/tikv-configuration-file.md b/tikv-configuration-file.md index c3d0c9c8c6b02..3cc182a0e29a8 100644 --- a/tikv-configuration-file.md +++ b/tikv-configuration-file.md @@ -1239,6 +1239,14 @@ Configuration items related to TiDB Lightning import and BR restore. + Default value: `8` + Minimum value: `1` +## gc + +### `enable-compaction-filter` + ++ 是否开启 GC in Compaction Filter 特性 ++ Default value: `true` + + ## backup Configuration items related to BR backup. From 6cfccce3e7be1aaa5de597c7126c4e59fc75965d Mon Sep 17 00:00:00 2001 From: Charlotte Liu <37295236+CharLotteiu@users.noreply.github.com> Date: Thu, 24 Dec 2020 18:09:03 +0800 Subject: [PATCH 2/7] refine words --- garbage-collection-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/garbage-collection-configuration.md b/garbage-collection-configuration.md index b973182ef119f..6717d7fa87537 100644 --- a/garbage-collection-configuration.md +++ b/garbage-collection-configuration.md @@ -146,7 +146,7 @@ tikv-ctl --host=ip:port modify-tikv-config -m server -n gc.max_write_bytes_per_s ## GC in Compaction Filter -From v5.0, TiKV introduces the mechanism of GC in Compaction Filter. Based on the distributed GC mode, the mechanism uses the compaction process of RocksDB, instead of a separate GC worker thread, to run GC. It helps to avoid extra disk read caused by GC. Also, after clearing the obsolete data, it prevents leaving behind a large number of deletion marks which degrade the sequential scan performance. This mechanism is enabled by default and is also automatically enabled after the rolling upgrade is completed. The following example shows how to enable this GC mechanism in the TiKV configuration file: +From v5.0, TiKV introduces the mechanism of GC in Compaction Filter. Based on the distributed GC mode, the mechanism uses the compaction process of RocksDB, instead of a separate GC worker thread, to run GC. It helps to avoid extra disk read caused by GC. Also, after clearing the obsolete data, it prevents leaving behind a large number of deletion marks which degrade the sequential scan performance. This mechanism is enabled by default and is also automatically enabled after a rolling upgrade. The following example shows how to enable this GC mechanism in the TiKV configuration file: {{< copyable "" >}} From 023224d43ac60c940c8b82d4a8e96d98fdc89b32 Mon Sep 17 00:00:00 2001 From: Charlotte Liu <37295236+CharLotteiu@users.noreply.github.com> Date: Thu, 24 Dec 2020 18:14:31 +0800 Subject: [PATCH 3/7] Update tikv-configuration-file.md --- tikv-configuration-file.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tikv-configuration-file.md b/tikv-configuration-file.md index 3cc182a0e29a8..577b77a4e4d68 100644 --- a/tikv-configuration-file.md +++ b/tikv-configuration-file.md @@ -1243,10 +1243,9 @@ Configuration items related to TiDB Lightning import and BR restore. ### `enable-compaction-filter` -+ 是否开启 GC in Compaction Filter 特性 ++ Controls whether to enable the GC in Compaction Filter feature + Default value: `true` - ## backup Configuration items related to BR backup. From cd347849ea5ea2e9807234423c3385bb03cd2a9f Mon Sep 17 00:00:00 2001 From: TomShawn <41534398+TomShawn@users.noreply.github.com> Date: Fri, 25 Dec 2020 19:18:50 +0800 Subject: [PATCH 4/7] Update garbage-collection-configuration.md --- garbage-collection-configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/garbage-collection-configuration.md b/garbage-collection-configuration.md index 6717d7fa87537..e347de11e8795 100644 --- a/garbage-collection-configuration.md +++ b/garbage-collection-configuration.md @@ -146,7 +146,7 @@ tikv-ctl --host=ip:port modify-tikv-config -m server -n gc.max_write_bytes_per_s ## GC in Compaction Filter -From v5.0, TiKV introduces the mechanism of GC in Compaction Filter. Based on the distributed GC mode, the mechanism uses the compaction process of RocksDB, instead of a separate GC worker thread, to run GC. It helps to avoid extra disk read caused by GC. Also, after clearing the obsolete data, it prevents leaving behind a large number of deletion marks which degrade the sequential scan performance. This mechanism is enabled by default and is also automatically enabled after a rolling upgrade. The following example shows how to enable this GC mechanism in the TiKV configuration file: +Since v5.0, TiDB introduces the mechanism of GC in Compaction Filter. Based on the distributed GC mode, the mechanism uses the compaction process of RocksDB, instead of a separate GC worker thread, to run GC. It helps to avoid extra disk read caused by GC. Also, after clearing the obsolete data, it avoids a large number of left tombstone marks which degrade the sequential scan performance. This mechanism is enabled by default and is also automatically enabled after a rolling upgrade. The following example shows how to enable this GC mechanism in the TiKV configuration file: {{< copyable "" >}} @@ -188,4 +188,4 @@ show config where type = 'tikv' and name like '%enable-compaction-filter%'; | tikv | 172.16.5.36:20163 | gc.enable-compaction-filter | true | | tikv | 172.16.5.35:20163 | gc.enable-compaction-filter | true | +------+-------------------+-----------------------------+-------+ -``` \ No newline at end of file +``` From 57aa85a074f5700f6210d2874239fbf0ea7f52a0 Mon Sep 17 00:00:00 2001 From: TomShawn <41534398+TomShawn@users.noreply.github.com> Date: Fri, 25 Dec 2020 19:18:58 +0800 Subject: [PATCH 5/7] Update garbage-collection-configuration.md --- garbage-collection-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/garbage-collection-configuration.md b/garbage-collection-configuration.md index e347de11e8795..f958fa76f89ca 100644 --- a/garbage-collection-configuration.md +++ b/garbage-collection-configuration.md @@ -74,7 +74,7 @@ update mysql.tidb set VARIABLE_VALUE="24h" where VARIABLE_NAME="tikv_gc_life_tim - `"distributed"` (default): Distributed GC mode. In the [Do GC](/garbage-collection-overview.md#do-gc) step, the GC leader on the TiDB side uploads the safe point to PD. Each TiKV node obtains the safe point respectively and performs GC on all leader Regions on the current node. This mode is supported from TiDB 3.0. - - `"central"`: Central GC mode. In the [Do GC](/garbage-collection-overview.md#do-gc) step, the GC leader sends GC requests to all Regions. This mode is adopted by TiDB 2.1 or earlier versions. Starting from TiKV 5.0, this mode is not supported. Clusters set to this mode automatically switch to the `distributed` mode. + - `"central"`: Central GC mode. In the [Do GC](/garbage-collection-overview.md#do-gc) step, the GC leader sends GC requests to all Regions. This mode is adopted by TiDB 2.1 or earlier versions. Starting from TiDB 5.0, this mode is not supported. Clusters set to this mode automatically switch to the `distributed` mode. ## `tikv_gc_auto_concurrency` From eed05adb1910002e991077a12bc57f20aeada8af Mon Sep 17 00:00:00 2001 From: TomShawn <41534398+TomShawn@users.noreply.github.com> Date: Mon, 28 Dec 2020 09:21:51 +0800 Subject: [PATCH 6/7] Update garbage-collection-configuration.md --- garbage-collection-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/garbage-collection-configuration.md b/garbage-collection-configuration.md index f958fa76f89ca..a4f2f7207f2b7 100644 --- a/garbage-collection-configuration.md +++ b/garbage-collection-configuration.md @@ -146,7 +146,7 @@ tikv-ctl --host=ip:port modify-tikv-config -m server -n gc.max_write_bytes_per_s ## GC in Compaction Filter -Since v5.0, TiDB introduces the mechanism of GC in Compaction Filter. Based on the distributed GC mode, the mechanism uses the compaction process of RocksDB, instead of a separate GC worker thread, to run GC. It helps to avoid extra disk read caused by GC. Also, after clearing the obsolete data, it avoids a large number of left tombstone marks which degrade the sequential scan performance. This mechanism is enabled by default and is also automatically enabled after a rolling upgrade. The following example shows how to enable this GC mechanism in the TiKV configuration file: +Since v5.0, TiDB introduces the mechanism of GC in Compaction Filter. Based on the distributed GC mode, the mechanism uses the compaction process of RocksDB, instead of a separate GC worker thread, to run GC. This new GC machanism helps to avoid extra disk read caused by GC. Also, after clearing the obsolete data, it avoids a large number of left tombstone marks which degrade the sequential scan performance. This mechanism is enabled by default and is also automatically enabled after a rolling upgrade. The following example shows how to enable this GC mechanism in the TiKV configuration file: {{< copyable "" >}} From 9f80681e83a153a1be564b576c9376b61aece5db Mon Sep 17 00:00:00 2001 From: TomShawn <41534398+TomShawn@users.noreply.github.com> Date: Mon, 28 Dec 2020 09:22:40 +0800 Subject: [PATCH 7/7] Update tikv-configuration-file.md --- tikv-configuration-file.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tikv-configuration-file.md b/tikv-configuration-file.md index 65aeb363145cf..ece8796bf26e0 100644 --- a/tikv-configuration-file.md +++ b/tikv-configuration-file.md @@ -1248,7 +1248,7 @@ Configuration items related to TiDB Lightning import and BR restore. ## gc -### `enable-compaction-filter` +### `enable-compaction-filter` + Controls whether to enable the GC in Compaction Filter feature + Default value: `true`