Skip to content

Commit

Permalink
*: add split table syntax to split table region (pingcap#10553)
Browse files Browse the repository at this point in the history
  • Loading branch information
@crazycs520
crazycs520 committed Jun 11, 2019
1 parent 479c733 commit 0dd47fc
Show file tree
Hide file tree
Showing 5 changed files with 347 additions and 45 deletions.
22 changes: 16 additions & 6 deletions executor/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor {
case *plannercore.SQLBindPlan:
return b.buildSQLBindExec(v)
case *plannercore.SplitRegion:
return b.buildSplitIndexRegion(v)
return b.buildSplitRegion(v)
default:
if mp, ok := p.(MockPhysicalPlan); ok {
return mp.GetExecutor()
Expand Down Expand Up @@ -1261,15 +1261,25 @@ func (b *executorBuilder) buildUnionAll(v *plannercore.PhysicalUnionAll) Executo
return e
}

func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitRegion) Executor {
func (b *executorBuilder) buildSplitRegion(v *plannercore.SplitRegion) Executor {
base := newBaseExecutor(b.ctx, nil, v.ExplainID())
base.initCap = chunk.ZeroCapacity
return &SplitIndexRegionExec{
if v.IndexInfo != nil {
return &SplitIndexRegionExec{
baseExecutor: base,
tableInfo: v.TableInfo,
indexInfo: v.IndexInfo,
lower: v.Lower,
upper: v.Upper,
num: v.Num,
valueLists: v.ValueLists,
}
}
return &SplitTableRegionExec{
baseExecutor: base,
tableInfo: v.TableInfo,
indexInfo: v.IndexInfo,
lower: v.Lower,
upper: v.Upper,
lower: v.Lower[0],
upper: v.Upper[0],
num: v.Num,
valueLists: v.ValueLists,
}
Expand Down
50 changes: 44 additions & 6 deletions executor/executor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3787,7 +3787,7 @@ func (s *testSuite) TestReadPartitionedTable(c *C) {
tk.MustQuery("select a from pt where b = 3").Check(testkit.Rows("3"))
}

func (s *testSuite) TestSplitIndexRegion(c *C) {
func (s *testSuite) TestSplitRegion(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
Expand All @@ -3798,26 +3798,27 @@ func (s *testSuite) TestSplitIndexRegion(c *C) {
terr := errors.Cause(err).(*terror.Error)
c.Assert(terr.Code(), Equals, terror.ErrCode(mysql.WarnDataTruncated))

// Test for split index region.
// Check min value is more than max value.
tk.MustExec(`split table t index idx1 between (0) and (1000000000) regions 10`)
_, err = tk.Exec(`split table t index idx1 between (2,'a') and (1,'c') regions 10`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "Split index region `idx1` lower value (2,a) should less than the upper value (1,c)")
c.Assert(err.Error(), Equals, "Split index `idx1` region lower value (2,a) should less than the upper value (1,c)")

// Check min value is invalid.
_, err = tk.Exec(`split table t index idx1 between () and (1) regions 10`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "Split index region `idx1` lower value count should more than 0")
c.Assert(err.Error(), Equals, "Split index `idx1` region lower value count should more than 0")

// Check max value is invalid.
_, err = tk.Exec(`split table t index idx1 between (1) and () regions 10`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "Split index region `idx1` upper value count should more than 0")
c.Assert(err.Error(), Equals, "Split index `idx1` region upper value count should more than 0")

// Check pre-split region num is too large.
_, err = tk.Exec(`split table t index idx1 between (0) and (1000000000) regions 10000`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "Split index region num is exceed the limit 1000")
c.Assert(err.Error(), Equals, "Split index region num exceeded the limit 1000")

// Check pre-split region num 0 is invalid.
_, err = tk.Exec(`split table t index idx1 between (0) and (1000000000) regions 0`)
Expand All @@ -3827,7 +3828,44 @@ func (s *testSuite) TestSplitIndexRegion(c *C) {
// Test truncate error msg.
_, err = tk.Exec(`split table t index idx1 between ("aa") and (1000000000) regions 0`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "[types:1265]Incorrect value: 'aa' for index column 'b'")
c.Assert(err.Error(), Equals, "[types:1265]Incorrect value: 'aa' for column 'b'")

// Test for split table region.
tk.MustExec(`split table t between (0) and (1000000000) regions 10`)
// Check the ower value is more than the upper value.
_, err = tk.Exec(`split table t between (2) and (1) regions 10`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "Split table `t` region lower value 2 should less than the upper value 1")

// Check the lower value is invalid.
_, err = tk.Exec(`split table t between () and (1) regions 10`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "Split table region lower value count should be 1")

// Check upper value is invalid.
_, err = tk.Exec(`split table t between (1) and () regions 10`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "Split table region upper value count should be 1")

// Check pre-split region num is too large.
_, err = tk.Exec(`split table t between (0) and (1000000000) regions 10000`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "Split table region num exceeded the limit 1000")

// Check pre-split region num 0 is invalid.
_, err = tk.Exec(`split table t between (0) and (1000000000) regions 0`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "Split table region num should more than 0")

// Test truncate error msg.
_, err = tk.Exec(`split table t between ("aa") and (1000000000) regions 10`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "[types:1265]Incorrect value: 'aa' for column '_tidb_rowid'")

// Test split table region step is too small.
_, err = tk.Exec(`split table t between (0) and (100) regions 10`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "Split table `t` region step value should more than 1000, step 10 is invalid")
}

func (s *testSuite) TestIssue10435(c *C) {
Expand Down
108 changes: 106 additions & 2 deletions executor/split.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/cznic/mathutil"
"github.com/pingcap/errors"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/tablecodec"
Expand Down Expand Up @@ -126,9 +127,9 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) {
lowerStr, err1 := datumSliceToString(e.lower)
upperStr, err2 := datumSliceToString(e.upper)
if err1 != nil || err2 != nil {
return nil, errors.Errorf("Split index region `%v` lower value %v should less than the upper value %v", e.indexInfo.Name, e.lower, e.upper)
return nil, errors.Errorf("Split index `%v` region lower value %v should less than the upper value %v", e.indexInfo.Name, e.lower, e.upper)
}
return nil, errors.Errorf("Split index region `%v` lower value %v should less than the upper value %v", e.indexInfo.Name, lowerStr, upperStr)
return nil, errors.Errorf("Split index `%v` region lower value %v should less than the upper value %v", e.indexInfo.Name, lowerStr, upperStr)
}
return getValuesList(lowerIdxKey, upperIdxKey, e.num, idxKeys), nil
}
Expand Down Expand Up @@ -204,3 +205,106 @@ func datumSliceToString(ds []types.Datum) (string, error) {
str += ")"
return str, nil
}

// SplitTableRegionExec represents a split table regions executor.
type SplitTableRegionExec struct {
baseExecutor

tableInfo *model.TableInfo
lower types.Datum
upper types.Datum
num int
valueLists [][]types.Datum
}

// Next implements the Executor Next interface.
func (e *SplitTableRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) error {
store := e.ctx.GetStore()
s, ok := store.(splitableStore)
if !ok {
return nil
}
splitKeys, err := e.getSplitTableKeys()
if err != nil {
return err
}
regionIDs := make([]uint64, 0, len(splitKeys))
for _, key := range splitKeys {
regionID, err := s.SplitRegionAndScatter(key)
if err != nil {
logutil.Logger(context.Background()).Warn("split table region failed",
zap.String("table", e.tableInfo.Name.L),
zap.Error(err))
continue
}
regionIDs = append(regionIDs, regionID)
}
if !e.ctx.GetSessionVars().WaitTableSplitFinish {
return nil
}
for _, regionID := range regionIDs {
err := s.WaitScatterRegionFinish(regionID)
if err != nil {
logutil.Logger(context.Background()).Warn("wait scatter region failed",
zap.Uint64("regionID", regionID),
zap.String("table", e.tableInfo.Name.L),
zap.Error(err))
}
}
return nil
}

var minRegionStepValue = uint64(1000)

func (e *SplitTableRegionExec) getSplitTableKeys() ([][]byte, error) {
var keys [][]byte
if e.num > 0 {
keys = make([][]byte, 0, e.num)
} else {
keys = make([][]byte, 0, len(e.valueLists))
}
recordPrefix := tablecodec.GenTableRecordPrefix(e.tableInfo.ID)
if len(e.valueLists) > 0 {
for _, v := range e.valueLists {
key := tablecodec.EncodeRecordKey(recordPrefix, v[0].GetInt64())
keys = append(keys, key)
}
return keys, nil
}
isUnsigned := false
if e.tableInfo.PKIsHandle {
if pkCol := e.tableInfo.GetPkColInfo(); pkCol != nil {
isUnsigned = mysql.HasUnsignedFlag(pkCol.Flag)
}
}
var step uint64
var lowerValue int64
if isUnsigned {
lowerRecordID := e.lower.GetUint64()
upperRecordID := e.upper.GetUint64()
if upperRecordID <= lowerRecordID {
return nil, errors.Errorf("Split table `%s` region lower value %v should less than the upper value %v", e.tableInfo.Name, lowerRecordID, upperRecordID)
}
step = (upperRecordID - lowerRecordID) / uint64(e.num)
lowerValue = int64(lowerRecordID)
} else {
lowerRecordID := e.lower.GetInt64()
upperRecordID := e.upper.GetInt64()
if upperRecordID <= lowerRecordID {
return nil, errors.Errorf("Split table `%s` region lower value %v should less than the upper value %v", e.tableInfo.Name, lowerRecordID, upperRecordID)
}
step = uint64(upperRecordID-lowerRecordID) / uint64(e.num)
lowerValue = lowerRecordID
}
if step < minRegionStepValue {
return nil, errors.Errorf("Split table `%s` region step value should more than %v, step %v is invalid", e.tableInfo.Name, minRegionStepValue, step)
}

recordID := lowerValue
for i := 1; i < e.num; i++ {
recordID += int64(step)
key := tablecodec.EncodeRecordKey(recordPrefix, recordID)
keys = append(keys, key)
}
return keys, nil
}
76 changes: 76 additions & 0 deletions executor/split_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/mock"
)
Expand Down Expand Up @@ -285,6 +286,81 @@ func (s *testSplitIndex) TestSplitIndex(c *C) {
}
}

func (s *testSplitIndex) TestSplitTable(c *C) {
tbInfo := &model.TableInfo{
Name: model.NewCIStr("t1"),
ID: rand.Int63(),
Columns: []*model.ColumnInfo{
{
Name: model.NewCIStr("c0"),
ID: 1,
Offset: 1,
DefaultValue: 0,
State: model.StatePublic,
FieldType: *types.NewFieldType(mysql.TypeLong),
},
},
}
defer func(originValue uint64) {
minRegionStepValue = originValue
}(minRegionStepValue)
minRegionStepValue = 10
// range is 0 ~ 100, and split into 10 region.
// So 10 regions range is like below:
// region1: [-inf ~ 10)
// region2: [10 ~ 20)
// region3: [20 ~ 30)
// region4: [30 ~ 40)
// region5: [40 ~ 50)
// region6: [50 ~ 60)
// region7: [60 ~ 70)
// region8: [70 ~ 80)
// region9: [80 ~ 90 )
// region10: [90 ~ +inf)
ctx := mock.NewContext()
e := &SplitTableRegionExec{
baseExecutor: newBaseExecutor(ctx, nil, nil),
tableInfo: tbInfo,
lower: types.NewDatum(0),
upper: types.NewDatum(100),
num: 10,
}
valueList, err := e.getSplitTableKeys()
c.Assert(err, IsNil)
c.Assert(len(valueList), Equals, e.num-1)

cases := []struct {
value int
lessEqualIdx int
}{
{-1, -1},
{0, -1},
{1, -1},
{10, 0},
{11, 0},
{20, 1},
{21, 1},
{31, 2},
{41, 3},
{51, 4},
{61, 5},
{71, 6},
{81, 7},
{91, 8},
{100, 8},
{1000, 8},
}

recordPrefix := tablecodec.GenTableRecordPrefix(e.tableInfo.ID)
for _, ca := range cases {
// test for minInt64 handle
key := tablecodec.EncodeRecordKey(recordPrefix, int64(ca.value))
c.Assert(err, IsNil)
idx := searchLessEqualIdx(valueList, key)
c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca))
}
}

func searchLessEqualIdx(valueList [][]byte, value []byte) int {
idx := -1
for i, v := range valueList {
Expand Down
Loading

0 comments on commit 0dd47fc

Please sign in to comment.