From d083fd6937ef1ee45cf779e2e7e889e72f61be92 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 3 Apr 2023 15:08:57 +0800 Subject: [PATCH] *: remove cluster dependency from keyspace (#6249) ref tikv/pd#6231 Signed-off-by: Ryan Leung Co-authored-by: Ti Chi Robot --- {server => pkg}/keyspace/keyspace.go | 42 +++++++++++-------- {server => pkg}/keyspace/keyspace_test.go | 14 +++++-- .../keyspace/tso_keyspace_group.go | 0 .../keyspace/tso_keyspace_group_test.go | 0 {server => pkg}/keyspace/util.go | 0 {server => pkg}/keyspace/util_test.go | 0 server/api/region_label_test.go | 4 +- server/apiv2/handlers/keyspace.go | 2 +- server/config/config.go | 5 +++ server/keyspace_service.go | 2 +- server/server.go | 12 +++--- tests/cluster.go | 2 +- tests/integrations/client/keyspace_test.go | 2 +- tests/server/apiv2/handlers/keyspace_test.go | 2 +- tests/server/keyspace/keyspace_test.go | 2 +- 15 files changed, 54 insertions(+), 35 deletions(-) rename {server => pkg}/keyspace/keyspace.go (94%) rename {server => pkg}/keyspace/keyspace_test.go (96%) rename {server => pkg}/keyspace/tso_keyspace_group.go (100%) rename {server => pkg}/keyspace/tso_keyspace_group_test.go (100%) rename {server => pkg}/keyspace/util.go (100%) rename {server => pkg}/keyspace/util_test.go (100%) diff --git a/server/keyspace/keyspace.go b/pkg/keyspace/keyspace.go similarity index 94% rename from server/keyspace/keyspace.go rename to pkg/keyspace/keyspace.go index bf64f3e680b..1d6fe227891 100644 --- a/server/keyspace/keyspace.go +++ b/pkg/keyspace/keyspace.go @@ -23,12 +23,12 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/id" + "github.com/tikv/pd/pkg/schedule" + "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/syncutil" - "github.com/tikv/pd/server/cluster" - "github.com/tikv/pd/server/config" "go.uber.org/zap" ) @@ -48,6 +48,11 @@ const ( regionLabelKey = "id" ) +// Config is the interface for keyspace config. +type Config interface { + GetPreAlloc() []string +} + // Manager manages keyspace related data. // It validates requests and provides concurrency control. type Manager struct { @@ -58,11 +63,11 @@ type Manager struct { // store is the storage for keyspace related information. store endpoint.KeyspaceStorage // rc is the raft cluster of the server. - rc *cluster.RaftCluster + cluster schedule.Cluster // ctx is the context of the manager, to be used in transaction. ctx context.Context // config is the configurations of the manager. - config config.KeyspaceConfig + config Config } // CreateKeyspaceRequest represents necessary arguments to create a keyspace. @@ -77,15 +82,15 @@ type CreateKeyspaceRequest struct { // NewKeyspaceManager creates a Manager of keyspace related data. func NewKeyspaceManager(store endpoint.KeyspaceStorage, - rc *cluster.RaftCluster, + cluster schedule.Cluster, idAllocator id.Allocator, - config config.KeyspaceConfig, + config Config, ) *Manager { return &Manager{ metaLock: syncutil.NewLockGroup(syncutil.WithHash(keyspaceIDHash)), idAllocator: idAllocator, store: store, - rc: rc, + cluster: cluster, ctx: context.TODO(), config: config, } @@ -113,7 +118,7 @@ func (manager *Manager) Bootstrap() error { } // Initialize pre-alloc keyspace. - preAlloc := manager.config.PreAlloc + preAlloc := manager.config.GetPreAlloc() for _, keyspaceName := range preAlloc { _, err = manager.CreateKeyspace(&CreateKeyspaceRequest{ Name: keyspaceName, @@ -207,18 +212,21 @@ func (manager *Manager) splitKeyspaceRegion(id uint32) error { }) keyspaceRule := makeLabelRule(id) - err := manager.rc.GetRegionLabeler().SetLabelRule(keyspaceRule) - if err != nil { - log.Warn("[keyspace] failed to add region label for keyspace", + if cl, ok := manager.cluster.(interface{ GetRegionLabeler() *labeler.RegionLabeler }); ok { + err := cl.GetRegionLabeler().SetLabelRule(keyspaceRule) + if err != nil { + log.Warn("[keyspace] failed to add region label for keyspace", + zap.Uint32("keyspaceID", id), + zap.Error(err), + ) + } + log.Info("[keyspace] added region label for keyspace", zap.Uint32("keyspaceID", id), - zap.Error(err), + zap.Any("LabelRule", keyspaceRule), ) + return nil } - log.Info("[keyspace] added region label for keyspace", - zap.Uint32("keyspaceID", id), - zap.Any("LabelRule", keyspaceRule), - ) - return nil + return errors.New("cluster does not support region label") } // LoadKeyspace returns the keyspace specified by name. diff --git a/server/keyspace/keyspace_test.go b/pkg/keyspace/keyspace_test.go similarity index 96% rename from server/keyspace/keyspace_test.go rename to pkg/keyspace/keyspace_test.go index d478736052e..1fc7252bc6a 100644 --- a/server/keyspace/keyspace_test.go +++ b/pkg/keyspace/keyspace_test.go @@ -29,7 +29,6 @@ import ( "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" - "github.com/tikv/pd/server/config" ) const ( @@ -47,18 +46,25 @@ func TestKeyspaceTestSuite(t *testing.T) { suite.Run(t, new(keyspaceTestSuite)) } +type mockConfig struct { + PreAlloc []string +} + +func (m *mockConfig) GetPreAlloc() []string { return m.PreAlloc } + func (suite *keyspaceTestSuite) SetupTest() { store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) allocator := mockid.NewIDAllocator() - suite.manager = NewKeyspaceManager(store, nil, allocator, config.KeyspaceConfig{}) + suite.manager = NewKeyspaceManager(store, nil, allocator, &mockConfig{}) suite.NoError(suite.manager.Bootstrap()) } func (suite *keyspaceTestSuite) SetupSuite() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/keyspace/skipSplitRegion", "return(true)")) + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) } + func (suite *keyspaceTestSuite) TearDownSuite() { - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/keyspace/skipSplitRegion")) + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) } func makeCreateKeyspaceRequests(count int) []*CreateKeyspaceRequest { diff --git a/server/keyspace/tso_keyspace_group.go b/pkg/keyspace/tso_keyspace_group.go similarity index 100% rename from server/keyspace/tso_keyspace_group.go rename to pkg/keyspace/tso_keyspace_group.go diff --git a/server/keyspace/tso_keyspace_group_test.go b/pkg/keyspace/tso_keyspace_group_test.go similarity index 100% rename from server/keyspace/tso_keyspace_group_test.go rename to pkg/keyspace/tso_keyspace_group_test.go diff --git a/server/keyspace/util.go b/pkg/keyspace/util.go similarity index 100% rename from server/keyspace/util.go rename to pkg/keyspace/util.go diff --git a/server/keyspace/util_test.go b/pkg/keyspace/util_test.go similarity index 100% rename from server/keyspace/util_test.go rename to pkg/keyspace/util_test.go diff --git a/server/api/region_label_test.go b/server/api/region_label_test.go index e377249561e..021ec7f1359 100644 --- a/server/api/region_label_test.go +++ b/server/api/region_label_test.go @@ -47,13 +47,13 @@ func (suite *regionLabelTestSuite) SetupSuite() { addr := suite.svr.GetAddr() suite.urlPrefix = fmt.Sprintf("%s%s/api/v1/config/region-label/", addr, apiPrefix) - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/keyspace/skipSplitRegion", "return(true)")) + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) mustBootstrapCluster(re, suite.svr) } func (suite *regionLabelTestSuite) TearDownSuite() { suite.cleanup() - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/keyspace/skipSplitRegion")) + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) } func (suite *regionLabelTestSuite) TestGetSet() { diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 1d607ab6017..2ac5235831b 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -25,9 +25,9 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/server" "github.com/tikv/pd/server/apiv2/middlewares" - "github.com/tikv/pd/server/keyspace" ) // RegisterKeyspace register keyspace related handlers to router paths. diff --git a/server/config/config.go b/server/config/config.go index 2142ad7ce41..070d6163266 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -1389,3 +1389,8 @@ type KeyspaceConfig struct { // PreAlloc contains the keyspace to be allocated during keyspace manager initialization. PreAlloc []string `toml:"pre-alloc" json:"pre-alloc"` } + +// GetPreAlloc returns the keyspace to be allocated during keyspace manager initialization. +func (c *KeyspaceConfig) GetPreAlloc() []string { + return c.PreAlloc +} diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 5fb06c38bdb..5255d725815 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -22,8 +22,8 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/storage/endpoint" - "github.com/tikv/pd/server/keyspace" "go.etcd.io/etcd/clientv3" ) diff --git a/server/server.go b/server/server.go index 7586cc23fe2..28568b2fcf5 100644 --- a/server/server.go +++ b/server/server.go @@ -46,6 +46,7 @@ import ( "github.com/tikv/pd/pkg/encryption" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/id" + "github.com/tikv/pd/pkg/keyspace" ms_server "github.com/tikv/pd/pkg/mcs/meta_storage/server" "github.com/tikv/pd/pkg/mcs/registry" rm_server "github.com/tikv/pd/pkg/mcs/resource_manager/server" @@ -73,7 +74,6 @@ import ( "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/gc" - "github.com/tikv/pd/server/keyspace" syncer "github.com/tikv/pd/server/region_syncer" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/embed" @@ -440,7 +440,7 @@ func (s *Server) startServer(ctx context.Context) error { Member: s.member.MemberValue(), Step: keyspace.AllocStep, }) - s.keyspaceManager = keyspace.NewKeyspaceManager(s.storage, s.cluster, keyspaceIDAllocator, s.cfg.Keyspace) + s.keyspaceManager = keyspace.NewKeyspaceManager(s.storage, s.cluster, keyspaceIDAllocator, &s.cfg.Keyspace) s.keyspaceGroupManager = keyspace.NewKeyspaceGroupManager(s.ctx, s.storage) s.hbStreams = hbstream.NewHeartbeatStreams(ctx, s.clusterID, s.cluster) // initial hot_region_storage in here. @@ -698,12 +698,12 @@ func (s *Server) bootstrapCluster(req *pdpb.BootstrapRequest) (*pdpb.BootstrapRe return nil, err } - if err = s.GetKeyspaceManager().Bootstrap(); err != nil { - log.Warn("bootstrap keyspace manager failed", errs.ZapError(err)) + if err := s.GetKeyspaceGroupManager().Bootstrap(); err != nil { + log.Warn("bootstrapping keyspace group manager failed", errs.ZapError(err)) } - if err = s.GetKeyspaceGroupManager().Bootstrap(); err != nil { - log.Warn("bootstrap keyspace group manager failed", errs.ZapError(err)) + if err = s.GetKeyspaceManager().Bootstrap(); err != nil { + log.Warn("bootstrapping keyspace manager failed", errs.ZapError(err)) } return &pdpb.BootstrapResponse{ diff --git a/tests/cluster.go b/tests/cluster.go index 1f27e72bcad..b0a5d529998 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -32,6 +32,7 @@ import ( "github.com/tikv/pd/pkg/dashboard" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/id" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/swaggerserver" @@ -44,7 +45,6 @@ import ( "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/join" - "github.com/tikv/pd/server/keyspace" "go.etcd.io/etcd/clientv3" ) diff --git a/tests/integrations/client/keyspace_test.go b/tests/integrations/client/keyspace_test.go index a9fb953cdf5..7cb35820bc6 100644 --- a/tests/integrations/client/keyspace_test.go +++ b/tests/integrations/client/keyspace_test.go @@ -20,9 +20,9 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/keyspace" ) const ( diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index e13e13737d7..b9b9742b2dc 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -26,9 +26,9 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/apiv2/handlers" - "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/tests" "go.uber.org/goleak" ) diff --git a/tests/server/keyspace/keyspace_test.go b/tests/server/keyspace/keyspace_test.go index 4f958d1f892..e108879f3c4 100644 --- a/tests/server/keyspace/keyspace_test.go +++ b/tests/server/keyspace/keyspace_test.go @@ -27,10 +27,10 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/codec" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/server/config" - "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/tests" )