Merge branch 'master' into moveRCCheck

DEPS.bzl

@@ -220,6 +220,14 @@ def go_deps():
         sum = "h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=",
         version = "v2.1.2",
     )
+    go_repository(
+        name = "com_github_charithe_durationcheck",
+        build_file_proto_mode = "disable",
+        importpath = "github.com/charithe/durationcheck",
+        sum = "h1:mPP4ucLrf/rKZiIG/a9IPXHGlh8p4CzgpyTy6EEutYk=",
+        version = "v0.0.9",
+    )
+
     go_repository(
         name = "com_github_cheggaaa_pb_v3",
         build_file_proto_mode = "disable_global",
@@ -1011,6 +1019,14 @@ def go_deps():
         sum = "h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=",
         version = "v0.0.0-20181017120253-0766667cb4d1",
     )
+    go_repository(
+        name = "com_github_gordonklaus_ineffassign",
+        build_file_proto_mode = "disable",
+        importpath = "github.com/gordonklaus/ineffassign",
+        sum = "h1:PVRE9d4AQKmbelZ7emNig1+NT27DUmKZn5qXxfio54U=",
+        version = "v0.0.0-20210914165742-4cc7213b9bc8",
+    )
+
     go_repository(
         name = "com_github_gorilla_handlers",
         build_file_proto_mode = "disable_global",
@@ -1541,6 +1557,14 @@ def go_deps():
         sum = "h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=",
         version = "v0.2.0",
     )
+    go_repository(
+        name = "com_github_kyoh86_exportloopref",
+        build_file_proto_mode = "disable",
+        importpath = "github.com/kyoh86/exportloopref",
+        sum = "h1:5Ry/at+eFdkX9Vsdw3qU4YkvGtzuVfzT4X7S77LoN/M=",
+        version = "v0.1.8",
+    )
+
     go_repository(
         name = "com_github_labstack_echo_v4",
         build_file_proto_mode = "disable_global",

br/pkg/backup/client.go

@@ -534,7 +534,9 @@ func (bc *Client) BackupRanges(
 	progressCallBack func(ProgressUnit),
 ) error {
 	init := time.Now()
-	defer log.Info("Backup Ranges", zap.Duration("take", time.Since(init)))
+	defer func() {
+		log.Info("Backup Ranges", zap.Duration("take", time.Since(init)))
+	}()
 
 	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
 		span1 := span.Tracer().StartSpan("Client.BackupRanges", opentracing.ChildOf(span.Context()))

br/pkg/lightning/BUILD.bazel

@@ -51,6 +51,7 @@ go_test(
         "//br/pkg/lightning/checkpoints",
         "//br/pkg/lightning/config",
         "//br/pkg/lightning/glue",
+        "//br/pkg/lightning/log",
         "//br/pkg/lightning/mydump",
         "//br/pkg/lightning/web",
         "@com_github_docker_go_units//:go-units",

br/pkg/lightning/errormanager/BUILD.bazel

@@ -26,6 +26,7 @@ go_test(
     embed = [":errormanager"],
     deps = [
         "//br/pkg/lightning/config",
+        "//br/pkg/lightning/log",
         "//br/pkg/utils",
         "@com_github_data_dog_go_sqlmock//:go-sqlmock",
         "@com_github_stretchr_testify//require",

br/pkg/task/restore_raw.go

@@ -146,8 +146,7 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR
 		!cfg.LogProgress)
 
 	// RawKV restore does not need to rewrite keys.
-	rewrite := &restore.RewriteRules{}
-	err = restore.SplitRanges(ctx, client, ranges, rewrite, updateCh, true)
+	err = restore.SplitRanges(ctx, client, ranges, nil, updateCh, true)
 	if err != nil {
 		return errors.Trace(err)
 	}

br/tests/br_rawkv/run.sh

@@ -98,12 +98,23 @@ run_test() {
         --key "$TEST_DIR/certs/br.key" \
         --mode put --put-data "311121:31, 31112100:32, 311122:33, 31112200:34, 3111220000:35, 311123:36"
 
+
+    # put some keys starts with t. https://github.com/pingcap/tidb/issues/35279
+    # t_128_r_12 --<hex encode>--> 745f3132385f725f3132
+    # t_128_r_13 --<hex encode>--> 745f3132385f725f3133
+    bin/rawkv --pd $PD_ADDR \
+        --ca "$TEST_DIR/certs/ca.pem" \
+        --cert "$TEST_DIR/certs/br.pem" \
+        --key "$TEST_DIR/certs/br.key" \
+        --mode put --put-data "745f3132385f725f3132:31, 745f3132385f725f3133:32"
+
     checksum_ori=$(checksum 31 3130303030303030)
     checksum_partial=$(checksum 311111 311122)
+    checksum_t_prefix=$(checksum 745f3132385f725f3131 745f3132385f725f3134)
 
     # backup rawkv
     echo "backup start..."
-    run_br --pd $PD_ADDR backup raw -s "local://$BACKUP_DIR" --start 31 --end 3130303030303030 --format hex --concurrency 4 --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef"
+    run_br --pd $PD_ADDR backup raw -s "local://$BACKUP_DIR" --start 31 --end 745f3132385f725f3134 --format hex --concurrency 4 --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef"
 
     # delete data in range[start-key, end-key)
     clean 31 3130303030303030
@@ -153,6 +164,21 @@ run_test() {
         fail_and_exit
     fi
 
+    echo "t prefix restore start..."
+    run_br --pd $PD_ADDR restore raw -s "local://$BACKUP_DIR" --start "745f3132385f725f3131" --end "745f3132385f725f3134" --format hex --concurrency 4 --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef"
+    bin/rawkv --pd $PD_ADDR \
+        --ca "$TEST_DIR/certs/ca.pem" \
+        --cert "$TEST_DIR/certs/br.pem" \
+        --key "$TEST_DIR/certs/br.key" \
+        --mode scan --start-key 745f3132385f725f3131 --end-key 745f3132385f725f3134
+
+    checksum_new=$(checksum 745f3132385f725f3131 745f3132385f725f3134)
+
+    if [ "$checksum_new" != "$checksum_t_prefix" ];then
+        echo "checksum failed after restore"
+        fail_and_exit
+    fi
+
     export GO_FAILPOINTS=""
 }
 

build/BUILD.bazel

@@ -81,6 +81,9 @@ nogo(
         "@org_golang_x_tools//go/analysis/passes/unreachable:go_default_library",
         "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_default_library",
         "@org_golang_x_tools//go/analysis/passes/unusedresult:go_default_library",
+        "//build/linter/durationcheck:durationcheck",
+        "//build/linter/exportloopref:exportloopref",
+        "//build/linter/ineffassign:ineffassign",
         "//build/linter/prealloc:prealloc",
     ] + staticcheck_analyzers(STATICHECK_ANALYZERS),
 )

build/linter/durationcheck/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "durationcheck",
+    srcs = ["analyzer.go"],
+    importpath = "github.com/pingcap/tidb/build/linter/durationcheck",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//build/linter/util",
+        "@com_github_charithe_durationcheck//:durationcheck",
+    ],
+)

build/linter/durationcheck/analyzer.go

@@ -0,0 +1,27 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package durationcheck
+
+import (
+	"github.com/charithe/durationcheck"
+	"github.com/pingcap/tidb/build/linter/util"
+)
+
+// Analyzer is the analyzer struct of durationcheck.
+var Analyzer = durationcheck.Analyzer
+
+func init() {
+	util.SkipAnalyzer(Analyzer)
+}

build/linter/exportloopref/BUILD.bazel

@@ -0,0 +1,9 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "exportloopref",
+    srcs = ["analyzer.go"],
+    importpath = "github.com/pingcap/tidb/build/linter/exportloopref",
+    visibility = ["//visibility:public"],
+    deps = ["@com_github_kyoh86_exportloopref//:exportloopref"],
+)

build/linter/exportloopref/analyzer.go

@@ -0,0 +1,20 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exportloopref
+
+import "github.com/kyoh86/exportloopref"
+
+// Analyzer is the analyzer struct of exportloopref.
+var Analyzer = exportloopref.Analyzer

build/linter/ineffassign/BUILD.bazel

@@ -0,0 +1,9 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "ineffassign",
+    srcs = ["analyzer.go"],
+    importpath = "github.com/pingcap/tidb/build/linter/ineffassign",
+    visibility = ["//visibility:public"],
+    deps = ["@com_github_gordonklaus_ineffassign//pkg/ineffassign"],
+)

build/linter/ineffassign/analyzer.go

@@ -0,0 +1,20 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ineffassign
+
+import "github.com/gordonklaus/ineffassign/pkg/ineffassign"
+
+// Analyzer is the analyzer struct of ineffassign.
+var Analyzer = ineffassign.Analyzer

build/linter/util/util.go

@@ -55,7 +55,7 @@ func parseDirective(s string) (cmd skipType, args []string) {
 		}
 		return skipNone, nil
 	}
-	s = strings.TrimPrefix(s, "//nolint: ")
+	s = strings.TrimPrefix(s, "//nolint:")
 	return skipLinter, []string{s}
 }
 

build/nogo_config.json

@@ -80,12 +80,24 @@
       ".*_generated\\.go$": "ignore generated code"
     }
   },
+  "durationcheck": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
   "errorsas": {
     "exclude_files": {
       "/external/": "no need to vet third party code",
       ".*_generated\\.go$": "ignore generated code"
     }
   },
+  "exportloopref": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
   "findcall": {
     "exclude_files": {
       "/external/": "no need to vet third party code",
@@ -104,6 +116,12 @@
       ".*_generated\\.go$": "ignore generated code"
     }
   },
+  "ineffassign": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
   "inspect": {
     "exclude_files": {
       "/external/": "no need to vet third party code",

config/config.go

@@ -117,6 +117,7 @@ var (
 			map[string]string{
 				"check-mb4-value-in-utf8":       "tidb_check_mb4_value_in_utf8",
 				"enable-collect-execution-info": "tidb_enable_collect_execution_info",
+				"max-server-connections":        "max_connections",
 			},
 		},
 		{
@@ -255,7 +256,8 @@ type Config struct {
 	// BallastObjectSize set the initial size of the ballast object, the unit is byte.
 	BallastObjectSize int `toml:"ballast-object-size" json:"ballast-object-size"`
 	// EnableGlobalKill indicates whether to enable global kill.
-	EnableGlobalKill bool `toml:"enable-global-kill" json:"enable-global-kill"`
+	EnableGlobalKill bool       `toml:"enable-global-kill" json:"enable-global-kill"`
+	TrxSummary       TrxSummary `toml:"transaction-summary" json:"transaction-summary"`
 
 	// The following items are deprecated. We need to keep them here temporarily
 	// to support the upgrade process. They can be removed in future.
@@ -473,6 +475,7 @@ type Instance struct {
 	EnableCollectExecutionInfo bool   `toml:"tidb_enable_collect_execution_info" json:"tidb_enable_collect_execution_info"`
 	PluginDir                  string `toml:"plugin_dir" json:"plugin_dir"`
 	PluginLoad                 string `toml:"plugin_load" json:"plugin_load"`
+	MaxConnections             uint32 `toml:"max_connections" json:"max_connections"`
 }
 
 func (l *Log) getDisableTimestamp() bool {
@@ -721,6 +724,22 @@ type PessimisticTxn struct {
 	PessimisticAutoCommit AtomicBool `toml:"pessimistic-auto-commit" json:"pessimistic-auto-commit"`
 }
 
+// TrxSummary is the config for transaction summary collecting.
+type TrxSummary struct {
+	// how many transaction summary in `transaction_summary` each TiDB node should keep.
+	TransactionSummaryCapacity uint `toml:"transaction-summary-capacity" json:"transaction-summary-capacity"`
+	// how long a transaction should be executed to make it be recorded in `transaction_id_digest`.
+	TransactionIDDigestMinDuration uint `toml:"transaction-id-digest-min-duration" json:"transaction-id-digest-min-duration"`
+}
+
+// Valid Validatse TrxSummary configs
+func (config *TrxSummary) Valid() error {
+	if config.TransactionSummaryCapacity > 5000 {
+		return errors.New("transaction-summary.transaction-summary-capacity should not be larger than 5000")
+	}
+	return nil
+}
+
 // DefaultPessimisticTxn returns the default configuration for PessimisticTxn
 func DefaultPessimisticTxn() PessimisticTxn {
 	return PessimisticTxn{
@@ -731,6 +750,15 @@ func DefaultPessimisticTxn() PessimisticTxn {
 	}
 }
 
+// DefaultTrxSummary returns the default configuration for TrxSummary collector
+func DefaultTrxSummary() TrxSummary {
+	// TrxSummary is not enabled by default before GA
+	return TrxSummary{
+		TransactionSummaryCapacity:     500,
+		TransactionIDDigestMinDuration: 2147483647,
+	}
+}
+
 // Plugin is the config for plugin
 type Plugin struct {
 	Dir  string `toml:"dir" json:"dir"`
@@ -824,6 +852,7 @@ var defaultConf = Config{
 		EnableCollectExecutionInfo:  true,
 		PluginDir:                   "/data/deploy/plugin",
 		PluginLoad:                  "",
+		MaxConnections:              0,
 	},
 	Status: Status{
 		ReportStatus:          true,
@@ -916,6 +945,7 @@ var defaultConf = Config{
 	EnableForwarding:                     defTiKVCfg.EnableForwarding,
 	NewCollationsEnabledOnFirstBootstrap: true,
 	EnableGlobalKill:                     true,
+	TrxSummary:                           DefaultTrxSummary(),
 }
 
 var (
@@ -1183,6 +1213,9 @@ func (c *Config) Valid() error {
 	if err := c.TiKVClient.Valid(); err != nil {
 		return err
 	}
+	if err := c.TrxSummary.Valid(); err != nil {
+		return err
+	}
 
 	if c.Performance.TxnTotalSizeLimit > 1<<40 {
 		return fmt.Errorf("txn-total-size-limit should be less than %d", 1<<40)

config/config.toml.example

@@ -89,9 +89,6 @@ repair-mode = false
 # In repair mode, repairing table which is not in repair list will get wrong database or wrong table error.
 repair-table-list = []
 
-# The maximum permitted number of simultaneous client connections. When the value is 0, the number of connections is unlimited.
-max-server-connections = 0
-
 # Whether new collations are enabled, as indicated by its name, this configuration entry take effect ONLY when a TiDB cluster bootstraps for the first time.
 new_collations_enabled_on_first_bootstrap = true
 
@@ -468,3 +465,6 @@ tidb_slow_log_threshold = 300
 # tidb_record_plan_in_slow_log is used to enable record query plan in slow log.
 # 0 is disable. 1 is enable.
 tidb_record_plan_in_slow_log = 1
+
+# The maximum permitted number of simultaneous client connections. When the value is 0, the number of connections is unlimited.
+max_connections = 0