-
Notifications
You must be signed in to change notification settings - Fork 721
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Summary of "keyspace group primary path change".
What:
The path for non-default keyspace group primary election changes from "/ms/{cluster_id}/tso/{group}/primary" to "/ms/{cluster_id}/tso/keyspace_groups/election/{group}/primary".
Default keyspace group keeps /ms/{cluster_id}/tso/00000/primary.
Why do we need this change?
We need to add watch loop to every TSO server to watch the primaries of all keyspace groups.
When #TSOServers > #GroupReplica, a TSO server only owns replicas of part of keyspace groups, i.e., the TSO server doesn't know the primaries of keyspace groups that the TSO server doesn't own.
The partial keyspace group state view leads to slow TSO service discovery and inconsistent membership view.
The above watch loop needs to watch the range [/ms/{cluster_id}/tso/00000/primary, /ms/{cluster_id}/tso/99999/primary],
but it overlaps with keyspace groups' timestamp path /ms/{cluster_id}/tso/{group}/{gta|lta}/.../timestamp which results in huge inefficiency (as far as I know, there is no built-in filter based on suffix).
Why do we keep the same path for Default keyspace group?
It has been deployed to prod, and the change will cause incompatability issues. Leave the change and upgrade to the future.
Signed-off-by: Bin Shi <[email protected]>- Loading branch information
1 parent
8b16b71
commit 8bbb45d
Showing
7 changed files
with
197 additions
and
78 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| // Copyright 2023 TiKV Project Authors. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package tso | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "regexp" | ||
| "strconv" | ||
| ) | ||
|
|
||
| // ExtractKeyspaceGroupIDFromPath extracts keyspace group id from the given path, which contains | ||
| // the pattern of `tso/keyspace_groups/membership/(\d{5})$`. | ||
| func ExtractKeyspaceGroupIDFromPath(compiledRegexp *regexp.Regexp, path string) (uint32, error) { | ||
| match := compiledRegexp.FindStringSubmatch(path) | ||
| if match == nil { | ||
| return 0, fmt.Errorf("invalid keyspace group id path: %s", path) | ||
| } | ||
| id, err := strconv.ParseUint(match[1], 10, 32) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("failed to parse keyspace group ID: %v", err) | ||
| } | ||
| return uint32(id), nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| // Copyright 2023 TiKV Project Authors. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package tso | ||
|
|
||
| import ( | ||
| "path" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| "github.com/tikv/pd/pkg/storage/endpoint" | ||
| ) | ||
|
|
||
| func TestExtractKeyspaceGroupIDFromKeyspaceGroupMembershipPath(t *testing.T) { | ||
| re := require.New(t) | ||
|
|
||
| compiledRegexp := endpoint.GetCompiledKeyspaceGroupIDRegexp() | ||
|
|
||
| rightCases := []struct { | ||
| path string | ||
| id uint32 | ||
| }{ | ||
| {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/00000", id: 0}, | ||
| {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/00001", id: 1}, | ||
| {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/12345", id: 12345}, | ||
| {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/99999", id: 99999}, | ||
| {path: "tso/keyspace_groups/membership/00000", id: 0}, | ||
| {path: "tso/keyspace_groups/membership/00001", id: 1}, | ||
| {path: "tso/keyspace_groups/membership/12345", id: 12345}, | ||
| {path: "tso/keyspace_groups/membership/99999", id: 99999}, | ||
| } | ||
|
|
||
| for _, tt := range rightCases { | ||
| id, err := ExtractKeyspaceGroupIDFromPath(compiledRegexp, tt.path) | ||
| re.Equal(tt.id, id) | ||
| re.NoError(err) | ||
| } | ||
|
|
||
| wrongCases := []struct { | ||
| path string | ||
| }{ | ||
| {path: ""}, | ||
| {path: "00001"}, | ||
| {path: "xxx/keyspace_groups/membership/00001"}, | ||
| {path: "tso/xxxxxxxxxxxxxxx/membership/00001"}, | ||
| {path: "tso/keyspace_groups/xxxxxxxxxx/00001"}, | ||
| {path: "/pd/{cluster_id}/tso/keyspace_groups/xxxxxxxxxx/00001"}, | ||
| {path: "/pd/{cluster_id}/xxx/keyspace_groups/membership/00001"}, | ||
| {path: "/pd/{cluster_id}/tso/xxxxxxxxxxxxxxx/membership/00001"}, | ||
| {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/"}, | ||
| {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/0"}, | ||
| {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/0001"}, | ||
| {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/123456"}, | ||
| {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/1234a"}, | ||
| {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/12345a"}, | ||
| } | ||
|
|
||
| for _, tt := range wrongCases { | ||
| _, err := ExtractKeyspaceGroupIDFromPath(compiledRegexp, tt.path) | ||
| re.Error(err) | ||
| } | ||
| } | ||
|
|
||
| func TestExtractKeyspaceGroupIDFromKeyspaceGroupPrimaryPath(t *testing.T) { | ||
| re := require.New(t) | ||
|
|
||
| tsoSvcRootPath := "/ms/111/tso" | ||
| primaryPathBuilder := &kgPrimaryPathBuilder{ | ||
| rootPath: tsoSvcRootPath, | ||
| defaultKeyspaceGroupIDPath: path.Join(tsoSvcRootPath, "00000"), | ||
| } | ||
|
|
||
| compiledRegexp := primaryPathBuilder.getCompiledNonDefaultIDRegexp() | ||
|
|
||
| rightCases := []struct { | ||
| path string | ||
| id uint32 | ||
| }{ | ||
| {path: "/ms/111/tso/keyspace_groups/election/00001/primary", id: 1}, | ||
| {path: "/ms/111/tso/keyspace_groups/election/12345/primary", id: 12345}, | ||
| {path: "/ms/111/tso/keyspace_groups/election/99999/primary", id: 99999}, | ||
| } | ||
|
|
||
| for _, tt := range rightCases { | ||
| id, err := ExtractKeyspaceGroupIDFromPath(compiledRegexp, tt.path) | ||
| re.Equal(tt.id, id) | ||
| re.NoError(err) | ||
| } | ||
| } |