Skip to content

Commit

Permalink
Merge branch 'master' into out2in
Browse files Browse the repository at this point in the history
  • Loading branch information
@ghazalfamilyusa
ghazalfamilyusa authored Apr 24, 2024
2 parents ffc25e9 + 3292909 commit 2515fe5
Show file tree
Hide file tree
Showing 70 changed files with 1,794 additions and 1,332 deletions.
12 changes: 6 additions & 6 deletions DEPS.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6807,13 +6807,13 @@ def go_deps():
name = "com_github_tikv_client_go_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/client-go/v2",
sha256 = "3a9d97649d1c917faebb2f7756e750213ab5fd34c070f17a38c9227b201862c9",
strip_prefix = "github.com/tikv/client-go/[email protected].20240409022718-714958ccd4d5",
sha256 = "9dc8899d26420c39a52014f920ba361e445f2600f613669f2b20a62dd5a56791",
strip_prefix = "github.com/tikv/client-go/[email protected].20240424052342-0229f4077f0c",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240409022718-714958ccd4d5.zip",
"http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240409022718-714958ccd4d5.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240409022718-714958ccd4d5.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240409022718-714958ccd4d5.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240424052342-0229f4077f0c.zip",
"http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240424052342-0229f4077f0c.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240424052342-0229f4077f0c.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240424052342-0229f4077f0c.zip",
],
)
go_repository(
Expand Down
3 changes: 2 additions & 1 deletion br/pkg/restore/import.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1340,7 +1340,8 @@ func (importer *FileImporter) ingestSSTs(
) (*import_sstpb.IngestResponse, error) {
leader := regionInfo.Leader
if leader == nil {
leader = regionInfo.Region.GetPeers()[0]
return nil, errors.Annotatef(berrors.ErrPDLeaderNotFound,
"region id %d has no leader", regionInfo.Region.Id)
}
reqCtx := &kvrpcpb.Context{
RegionId: regionInfo.Region.GetId(),
Expand Down
11 changes: 7 additions & 4 deletions br/pkg/restore/import_retry_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ func initTestClient(isRawKv bool) *TestClient {
}
regions[i] = &split.RegionInfo{
Leader: &metapb.Peer{
Id: i,
Id: i,
StoreId: 1,
},
Region: &metapb.Region{
Id: i,
Expand Down Expand Up @@ -281,7 +282,7 @@ func TestEpochNotMatch(t *testing.T) {
{Id: 43},
},
},
Leader: &metapb.Peer{Id: 43},
Leader: &metapb.Peer{Id: 43, StoreId: 1},
}
newRegion := pdtypes.NewRegionInfo(info.Region, info.Leader)
mergeRegion := func() {
Expand Down Expand Up @@ -340,7 +341,8 @@ func TestRegionSplit(t *testing.T) {
EndKey: codec.EncodeBytes(nil, []byte("aayy")),
},
Leader: &metapb.Peer{
Id: 43,
Id: 43,
StoreId: 1,
},
},
{
Expand All @@ -350,7 +352,8 @@ func TestRegionSplit(t *testing.T) {
EndKey: target.Region.EndKey,
},
Leader: &metapb.Peer{
Id: 45,
Id: 45,
StoreId: 1,
},
},
}
Expand Down
7 changes: 6 additions & 1 deletion br/pkg/restore/split/mock_pd_client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,13 @@ func (c *MockPDClientForSplit) setRegions(boundaries [][]byte) []*metapb.Region
StartKey: boundaries[i-1],
EndKey: boundaries[i],
}
p := &metapb.Peer{
Id: c.lastRegionID,
StoreId: 1,
}
c.Regions.SetRegion(&pdtypes.Region{
Meta: r,
Meta: r,
Leader: p,
})
ret = append(ret, r)
}
Expand Down
16 changes: 16 additions & 0 deletions br/pkg/restore/split/split.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,23 @@ func checkRegionConsistency(startKey, endKey []byte, regions []*RegionInfo) erro
}

cur := regions[0]
if cur.Leader == nil {
return errors.Annotatef(berrors.ErrPDBatchScanRegion,
"region %d's leader is nil", cur.Region.Id)
}
if cur.Leader.StoreId == 0 {
return errors.Annotatef(berrors.ErrPDBatchScanRegion,
"region %d's leader's store id is 0", cur.Region.Id)
}
for _, r := range regions[1:] {
if r.Leader == nil {
return errors.Annotatef(berrors.ErrPDBatchScanRegion,
"region %d's leader is nil", r.Region.Id)
}
if r.Leader.StoreId == 0 {
return errors.Annotatef(berrors.ErrPDBatchScanRegion,
"region %d's leader's store id is 0", r.Region.Id)
}
if !bytes.Equal(cur.Region.EndKey, r.Region.StartKey) {
return errors.Annotatef(berrors.ErrPDBatchScanRegion,
"region %d's endKey not equal to next region %d's startKey, endKey: %s, startKey: %s, region epoch: %s %s",
Expand Down
76 changes: 76 additions & 0 deletions br/pkg/restore/split/split_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -504,13 +504,21 @@ func TestPaginateScanRegion(t *testing.T) {
StartKey: []byte{1},
EndKey: []byte{2},
},
Leader: &metapb.Peer{
Id: 1,
StoreId: 1,
},
})
mockPDClient.Regions.SetRegion(&pdtypes.Region{
Meta: &metapb.Region{
Id: 4,
StartKey: []byte{4},
EndKey: []byte{5},
},
Leader: &metapb.Peer{
Id: 4,
StoreId: 1,
},
})

_, err = PaginateScanRegion(ctx, mockClient, []byte{1}, []byte{5}, 3)
Expand All @@ -525,13 +533,21 @@ func TestPaginateScanRegion(t *testing.T) {
StartKey: []byte{2},
EndKey: []byte{3},
},
Leader: &metapb.Peer{
Id: 2,
StoreId: 1,
},
},
{
Meta: &metapb.Region{
Id: 3,
StartKey: []byte{3},
EndKey: []byte{4},
},
Leader: &metapb.Peer{
Id: 3,
StoreId: 1,
},
},
}
mockPDClient.scanRegions.beforeHook = func() {
Expand Down Expand Up @@ -590,6 +606,10 @@ func TestRegionConsistency(t *testing.T) {
"region 6's endKey not equal to next region 8's startKey(.*?)",
[]*RegionInfo{
{
Leader: &metapb.Peer{
Id: 6,
StoreId: 1,
},
Region: &metapb.Region{
Id: 6,
StartKey: codec.EncodeBytes([]byte{}, []byte("b")),
Expand All @@ -598,6 +618,10 @@ func TestRegionConsistency(t *testing.T) {
},
},
{
Leader: &metapb.Peer{
Id: 8,
StoreId: 1,
},
Region: &metapb.Region{
Id: 8,
StartKey: codec.EncodeBytes([]byte{}, []byte("e")),
Expand All @@ -606,6 +630,58 @@ func TestRegionConsistency(t *testing.T) {
},
},
},
{
codec.EncodeBytes([]byte{}, []byte("c")),
codec.EncodeBytes([]byte{}, []byte("e")),
"region 6's leader is nil(.*?)",
[]*RegionInfo{
{
Region: &metapb.Region{
Id: 6,
StartKey: codec.EncodeBytes([]byte{}, []byte("c")),
EndKey: codec.EncodeBytes([]byte{}, []byte("d")),
RegionEpoch: nil,
},
},
{
Region: &metapb.Region{
Id: 8,
StartKey: codec.EncodeBytes([]byte{}, []byte("d")),
EndKey: codec.EncodeBytes([]byte{}, []byte("e")),
},
},
},
},
{
codec.EncodeBytes([]byte{}, []byte("c")),
codec.EncodeBytes([]byte{}, []byte("e")),
"region 6's leader's store id is 0(.*?)",
[]*RegionInfo{
{
Leader: &metapb.Peer{
Id: 6,
StoreId: 0,
},
Region: &metapb.Region{
Id: 6,
StartKey: codec.EncodeBytes([]byte{}, []byte("c")),
EndKey: codec.EncodeBytes([]byte{}, []byte("d")),
RegionEpoch: nil,
},
},
{
Leader: &metapb.Peer{
Id: 6,
StoreId: 0,
},
Region: &metapb.Region{
Id: 8,
StartKey: codec.EncodeBytes([]byte{}, []byte("d")),
EndKey: codec.EncodeBytes([]byte{}, []byte("e")),
},
},
},
},
}
for _, ca := range cases {
err := checkRegionConsistency(ca.startKey, ca.endKey, ca.regions)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ require (
github.com/tdakkota/asciicheck v0.2.0
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
github.com/tidwall/btree v1.7.0
github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5
github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c
github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e
github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a
github.com/twmb/murmur3 v1.1.6
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -785,8 +785,8 @@ github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 h1:NIYXG5l8JrDyc7k0zO17ppKJkRlUfKMWFOnjIQT5Tc4=
github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5/go.mod h1:+vXk4Aex17GnI8gfSMPxrL0SQLbBYgP3Db4FvHiImwM=
github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c h1:M97Y/RO0vGpX0FplwGTk02idZDmSPEJlO6fTCPaxkCI=
github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c/go.mod h1:+vXk4Aex17GnI8gfSMPxrL0SQLbBYgP3Db4FvHiImwM=
github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e h1:u2OoEvmh3qyjIiAKXUPRiFCOSwznByMINDx2fsorjAo=
github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e/go.mod h1:Z/QAgOt29zvwBTd0H6pdx45VO6KRNc/O/DzGkVmSyZg=
github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo=
Expand Down
2 changes: 0 additions & 2 deletions lightning/pkg/importer/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ go_library(
"//pkg/meta/autoid",
"//pkg/parser",
"//pkg/parser/ast",
"//pkg/parser/format",
"//pkg/parser/model",
"//pkg/parser/mysql",
"//pkg/planner/core",
Expand Down Expand Up @@ -116,7 +115,6 @@ go_test(
"meta_manager_test.go",
"precheck_impl_test.go",
"precheck_test.go",
"restore_schema_test.go",
"table_import_test.go",
"tidb_test.go",
],
Expand Down
Loading

0 comments on commit 2515fe5

Please sign in to comment.