lightning: init client-go global cfg (#45464) (#45467)

close #45462
pingcap · Jul 19, 2023 · cf44157 · cf44157
1 parent c514581
commit cf44157
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 1 deletion.
diff --git a/br/pkg/lightning/importer/BUILD.bazel b/br/pkg/lightning/importer/BUILD.bazel
@@ -44,6 +44,7 @@ go_library(
         "//br/pkg/utils",
         "//br/pkg/version",
         "//br/pkg/version/build",
+        "//config",
         "//ddl",
         "//errno",
         "//keyspace",
@@ -78,6 +79,7 @@ go_library(
         "@com_github_pingcap_kvproto//pkg/import_sstpb",
         "@com_github_pingcap_kvproto//pkg/metapb",
         "@com_github_prometheus_client_golang//prometheus",
+        "@com_github_tikv_client_go_v2//config",
         "@com_github_tikv_pd_client//:client",
         "@io_etcd_go_etcd_client_v3//:client",
         "@org_golang_google_grpc//:grpc",
@@ -107,7 +109,7 @@ go_test(
     ],
     embed = [":importer"],
     flaky = True,
-    shard_count = 49,
+    shard_count = 50,
     deps = [
         "//br/pkg/lightning/backend",
         "//br/pkg/lightning/backend/encode",
@@ -159,6 +161,7 @@ go_test(
         "@com_github_pingcap_kvproto//pkg/metapb",
         "@com_github_stretchr_testify//require",
         "@com_github_stretchr_testify//suite",
+        "@com_github_tikv_client_go_v2//config",
         "@com_github_xitongsys_parquet_go//writer",
         "@com_github_xitongsys_parquet_go_source//buffer",
         "@io_etcd_go_etcd_client_v3//:client",

diff --git a/br/pkg/lightning/importer/import.go b/br/pkg/lightning/importer/import.go
@@ -52,6 +52,7 @@ import (
 	"github.com/pingcap/tidb/br/pkg/utils"
 	"github.com/pingcap/tidb/br/pkg/version"
 	"github.com/pingcap/tidb/br/pkg/version/build"
+	tidbconfig "github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/errno"
 	tidbkv "github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/meta/autoid"
@@ -65,6 +66,7 @@ import (
 	regexprrouter "github.com/pingcap/tidb/util/regexpr-router"
 	"github.com/pingcap/tidb/util/set"
 	"github.com/prometheus/client_golang/prometheus"
+	tikvconfig "github.com/tikv/client-go/v2/config"
 	pd "github.com/tikv/pd/client"
 	"go.uber.org/atomic"
 	"go.uber.org/multierr"
@@ -357,6 +359,8 @@ func NewImportControllerWithPauser(
 			}
 		}
 
+		initGlobalConfig(tls.ToTiKVSecurityConfig())
+
 		encodingBuilder = local.NewEncodingBuilder(ctx)
 		regionSizeGetter := &local.TableRegionSizeGetterImpl{
 			DB: db,
@@ -2358,3 +2362,18 @@ func filterColumns(columnNames []string, extendData mydump.ExtendColumnData, ign
 	}
 	return filteredColumns, extendValueDatums
 }
+
+// check store liveness of tikv client-go requires GlobalConfig to work correctly, so we need to init it,
+// else tikv will report SSL error when tls is enabled.
+// and the SSL error seems affects normal logic of newer TiKV version, and cause the error "tikv: region is unavailable"
+// during checksum.
+// todo: DM relay on lightning physical mode too, but client-go doesn't support passing TLS data as bytes,
+func initGlobalConfig(secCfg tikvconfig.Security) {
+	if secCfg.ClusterSSLCA != "" || secCfg.ClusterSSLCert != "" {
+		conf := tidbconfig.GetGlobalConfig()
+		conf.Security.ClusterSSLCA = secCfg.ClusterSSLCA
+		conf.Security.ClusterSSLCert = secCfg.ClusterSSLCert
+		conf.Security.ClusterSSLKey = secCfg.ClusterSSLKey
+		tidbconfig.StoreGlobalConfig(conf)
+	}
+}
diff --git a/br/pkg/lightning/importer/import_test.go b/br/pkg/lightning/importer/import_test.go
@@ -38,6 +38,7 @@ import (
 	tmock "github.com/pingcap/tidb/util/mock"
 	router "github.com/pingcap/tidb/util/table-router"
 	"github.com/stretchr/testify/require"
+	tikvconfig "github.com/tikv/client-go/v2/config"
 )
 
 func TestNewTableRestore(t *testing.T) {
@@ -415,3 +416,29 @@ func TestFilterColumns(t *testing.T) {
 		require.Equal(t, expectedDatums, extendDatums)
 	}
 }
+
+func TestInitGlobalConfig(t *testing.T) {
+	require.Empty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLCA)
+	require.Empty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLCert)
+	require.Empty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLKey)
+	initGlobalConfig(tikvconfig.Security{})
+	require.Empty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLCA)
+	require.Empty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLCert)
+	require.Empty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLKey)
+
+	initGlobalConfig(tikvconfig.Security{
+		ClusterSSLCA: "ca",
+	})
+	require.NotEmpty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLCA)
+	require.Empty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLCert)
+	require.Empty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLKey)
+
+	initGlobalConfig(tikvconfig.Security{})
+	initGlobalConfig(tikvconfig.Security{
+		ClusterSSLCert: "cert",
+		ClusterSSLKey:  "key",
+	})
+	require.Empty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLCA)
+	require.NotEmpty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLCert)
+	require.NotEmpty(t, tikvconfig.GetGlobalConfig().Security.ClusterSSLKey)
+}