From 5c613add39a31738a27fee865a70046c7dc7c732 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 10 Apr 2019 13:07:27 +0800 Subject: [PATCH 01/22] init --- ddl/table.go | 32 +++++++++++++++++++++++ store/tikv/split_region.go | 53 +++++++++++++++++++++++++++++++++----- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/ddl/table.go b/ddl/table.go index 7cb63bf6e6915..fe88708b6e29c 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -74,6 +74,11 @@ func onCreateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) // TODO: Add restrictions to this operation. go splitTableRegion(d.store, tbInfo.ID) } + + if tbInfo.ShardRowIDBits > 0 { + go preSplitTableRegion(d.store, tbInfo, tbInfo.ShardRowIDBits/2) + } + // Finish this job. job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo) asyncNotifyEvent(d, &util.Event{Tp: model.ActionCreateTable, TableInfo: tbInfo}) @@ -321,6 +326,7 @@ func checkSafePoint(w *worker, snapshotTS uint64) error { type splitableStore interface { SplitRegion(splitKey kv.Key) error + SplitRegionAndScatter(splitKey kv.Key) error } func splitTableRegion(store kv.Storage, tableID int64) { @@ -335,6 +341,32 @@ func splitTableRegion(store kv.Storage, tableID int64) { } } +func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo, shardBits uint64) { + s, ok := store.(splitableStore) + if !ok { + return + } + // split table region + step := int64(1 << (tblInfo.ShardRowIDBits - shardBits)) + max := int64(1 << tblInfo.ShardRowIDBits) + for p := int64(step); p < max; p += step { + recordID := p << (64 - tblInfo.ShardRowIDBits) + recordPrefix := tablecodec.GenTableRecordPrefix(tblInfo.ID) + key := tablecodec.EncodeRecordKey(recordPrefix, recordID) + if err := s.SplitRegionAndScatter(key); err != nil { + logutil.Logger(ddlLogCtx).Warn("[ddl] split table region failed", zap.Error(err)) + } + } + + // split index region. + for _, idx := range tblInfo.Indices { + indexPrefix := tablecodec.EncodeTableIndexPrefix(tblInfo.ID, idx.ID) + if err := s.SplitRegionAndScatter(indexPrefix); err != nil { + logutil.Logger(ddlLogCtx).Warn("[ddl] split table region failed", zap.Error(err)) + } + } +} + func getTable(store kv.Storage, schemaID int64, tblInfo *model.TableInfo) (table.Table, error) { alloc := autoid.NewAllocator(store, tblInfo.GetDBID(schemaID), tblInfo.IsAutoIncColUnsigned()) tbl, err := table.TableFromMeta(alloc, tblInfo) diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index 985545a80ef28..44d3d785c9e6f 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -16,9 +16,9 @@ package tikv import ( "bytes" "context" - "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/util/logutil" @@ -28,6 +28,11 @@ import ( // SplitRegion splits the region contains splitKey into 2 regions: [start, // splitKey) and [splitKey, end). func (s *tikvStore) SplitRegion(splitKey kv.Key) error { + _, err := s.splitRegion(splitKey) + return err +} + +func (s *tikvStore) splitRegion(splitKey kv.Key) (*metapb.Region, error) { logutil.Logger(context.Background()).Info("start split region", zap.Binary("at", splitKey)) bo := NewBackoffer(context.Background(), splitRegionBackoff) @@ -42,25 +47,25 @@ func (s *tikvStore) SplitRegion(splitKey kv.Key) error { for { loc, err := s.regionCache.LocateKey(bo, splitKey) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } if bytes.Equal(splitKey, loc.StartKey) { logutil.Logger(context.Background()).Info("skip split region", zap.Binary("at", splitKey)) - return nil + return nil, nil } res, err := sender.SendReq(bo, req, loc.Region, readTimeoutShort) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } regionErr, err := res.GetRegionError() if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } if regionErr != nil { err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } continue } @@ -68,6 +73,42 @@ func (s *tikvStore) SplitRegion(splitKey kv.Key) error { zap.Binary("at", splitKey), zap.Stringer("new region left", res.SplitRegion.GetLeft()), zap.Stringer("new region right", res.SplitRegion.GetRight())) + return res.SplitRegion.GetLeft(), nil + } +} + +/* +func (s *tikvStore) scatterRegion(left *metapb.Region) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + resp3, err := s.pdClient.ScatterRegion(ctx, &pdpb.ScatterRegionRequest{ + Header: &pdpb.RequestHeader{ + ClusterId: s.clusterID, + }, + RegionId: left.GetId(), + }) + cancel() + + if err != nil { + return err + } + if resp3.Header.GetError() != nil { + return errors.New("scatter region failed") + } + return nil + +} +*/ + +func (s *tikvStore) SplitRegionAndScatter(splitKey kv.Key) error { + logutil.Logger(context.Background()).Info("split region and scatter\n\n", zap.Binary("key", splitKey)) + left, err := s.splitRegion(splitKey) + if err != nil { + return err + } + if left == nil { return nil } + + return nil + } From 2bc50c8ebd0e255d1f293c25c69520ab8d44ac64 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Fri, 12 Apr 2019 16:23:05 +0800 Subject: [PATCH 02/22] add scatter --- go.mod | 18 ++------- go.sum | 69 ++++++++++++++++++++++++++++++++++ store/mockstore/mocktikv/pd.go | 11 +++++- store/tikv/split_region.go | 25 +++--------- 4 files changed, 88 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index ca535624af6d6..a3ec2ca3ceda3 100644 --- a/go.mod +++ b/go.mod @@ -3,36 +3,28 @@ module github.com/pingcap/tidb require ( github.com/BurntSushi/toml v0.3.1 github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect - github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d github.com/boltdb/bolt v1.3.1 // indirect github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/coreos/bbolt v1.3.0 // indirect github.com/coreos/etcd v3.3.10+incompatible - github.com/coreos/go-semver v0.2.0 // indirect github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a // indirect - github.com/ghodss/yaml v1.0.0 // indirect github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4 github.com/gogo/protobuf v1.2.0 // indirect - github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff // indirect github.com/golang/protobuf v1.2.0 - github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/mux v1.6.2 github.com/gorilla/websocket v1.4.0 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 + github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway v1.5.1 // indirect - github.com/jonboulle/clockwork v0.1.0 // indirect github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect @@ -49,7 +41,7 @@ require ( github.com/pingcap/errors v0.11.1 github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3 github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e - github.com/pingcap/kvproto v0.0.0-20190215154024-7f2fc73ef562 + github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 github.com/pingcap/parser v0.0.0-20190409044748-a0b301443a30 github.com/pingcap/pd v2.1.0-rc.4+incompatible @@ -65,19 +57,15 @@ require ( github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371 // indirect github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca // indirect github.com/sirupsen/logrus v1.2.0 - github.com/soheilhy/cmux v0.1.4 // indirect github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 github.com/struCoder/pidusage v0.1.2 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 github.com/tinylib/msgp v1.1.0 // indirect - github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect github.com/twinj/uuid v1.0.0 github.com/uber-go/atomic v1.3.2 // indirect github.com/uber/jaeger-client-go v2.15.0+incompatible github.com/uber/jaeger-lib v1.5.0 // indirect - github.com/ugorji/go v0.0.0-20171019201919-bdcc60b419d1 // indirect github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d // indirect - github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect go.uber.org/zap v1.9.1 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb // indirect @@ -90,3 +78,5 @@ require ( sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) + +replace github.com/pingcap/pd => github.com/nolouch/pd v0.0.0-20190411093849-8f8de63fdb9f diff --git a/go.sum b/go.sum index 6cf70f55d8c18..cdd0ca7e74881 100644 --- a/go.sum +++ b/go.sum @@ -5,10 +5,14 @@ github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8 github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -19,8 +23,10 @@ github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= @@ -32,25 +38,31 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a h1:QNEenQIsGDEEfFNSnN+h6hE1OwnHqTg7Dl9gEk1Cko4= github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4 h1:3DFRjZdCDhzvxDf0U6/1qAryzOqD7Y5iAj0DJRRl1bs= github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= @@ -60,33 +72,54 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.5.1 h1:3scN4iuXkNOyP98jF55Lv8a9j1o/IwvnDIZ0LHJK1nk= github.com/grpc-ecosystem/grpc-gateway v1.5.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe h1:W/GaMY0y69G4cFlmsC6B9sbuo2fP8OFP1ABjt4kPz+w= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 h1:pmpDGKLw4n82EtrNiLqB+xSz/JQwFOaZuMALYUHwX5s= github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/myesui/uuid v1.0.0 h1:xCBmH4l5KuvLYc5L7AS7SZg9/jKdIFubM7OVoLqaQUI= @@ -95,9 +128,13 @@ github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdc github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= +github.com/nolouch/pd v0.0.0-20190411093849-8f8de63fdb9f h1:ys7t3nE/ycGCknGGctRh4l2r2KloucokCxyjwOLrkxQ= +github.com/nolouch/pd v0.0.0-20190411093849-8f8de63fdb9f/go.mod h1:MUCxRzOkYiWZtlyi4MhxjCIj9PgQQ/j+BLNGm7aUsnM= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= @@ -108,6 +145,8 @@ github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= +github.com/pingcap/errors v0.10.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.1 h1:BXFZ6MdDd2U1uJUa2sRAWTmm+nieEzuyYM0R4aUTcC8= github.com/pingcap/errors v0.11.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -117,6 +156,9 @@ github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rG github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20190215154024-7f2fc73ef562 h1:32oF1/8lVnBR2JVcCAnKPQATTOX0+ckRDFpjQk4Ngno= github.com/pingcap/kvproto v0.0.0-20190215154024-7f2fc73ef562/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= +github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d h1:LJYJl+cBhkkTWD79n+n9Bp4agQ85SdF9YKY4zEtL9Kw= +github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= +github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= github.com/pingcap/parser v0.0.0-20190409044748-a0b301443a30 h1:Cu+VJBHLUqI0TFj/0Kya4L1iHIJZ3VbtZcEwv+3zOxQ= @@ -127,18 +169,24 @@ github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190107072121-abbec73437b7 h1:wnjdQRhybddDesBVBKyOLUPgDaOFdtqA92pduBgWvVQ= github.com/pingcap/tipb v0.0.0-20190107072121-abbec73437b7/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 h1:gGBSHPOU7g8YjTbhwn+lvFm2VDEhhA+PwDIlstkgSxE= github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.0 h1:tXuTFVHC03mW0D+Ua1Q2d1EAVqLTuggX50V0VLICCzY= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 h1:Cto4X6SVMWRPBkJ/3YHn1iDGDGc/Z+sW+AEMKHMVvN4= github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d h1:GoAlyOgbOEIFdaDqxJVlbOQ1DtGmZWs/Qau0hIlk+WQ= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA= @@ -149,12 +197,15 @@ github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371 h1:SWV2fHctRpRrp49 github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca h1:3fECS8atRjByijiI8yYiuwLwQ2ZxXobW7ua/8GRB3pI= github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -162,10 +213,12 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/struCoder/pidusage v0.1.2 h1:fFPTThlcWFQyizv3xKs5Lyq1lpG5lZ36arEGNhWz2Vs= github.com/struCoder/pidusage v0.1.2/go.mod h1:pWBlW3YuSwRl6h7R5KbvA4N8oOqe9LjaKW5CwT1SPjI= +github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twinj/uuid v1.0.0 h1:fzz7COZnDrXGTAOHGuUGYd6sG+JMq+AoE7+Jlu0przk= @@ -178,16 +231,25 @@ github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v0.0.0-20171019201919-bdcc60b419d1 h1:UvhxfNjNqlZ/x3cDyqxMhoiUpemd3zXkVQApN6bM/lg= github.com/ugorji/go v0.0.0-20171019201919-bdcc60b419d1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= +github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d h1:ggUgChAeyge4NZ4QUw6lhHsVymzwSDJOZcE0s2X8S20= github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20190320044326-77d4b742cdbf/go.mod h1:KSGwdbiFchh5KIC9My2+ZVl5/3ANcwohw50dpPwa2cw= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -209,24 +271,31 @@ golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYb golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190130214255-bb1329dc71a0 h1:iRpjPej1fPzmfoBhMFkp3HdqzF+ytPmAwiQhJGV0zGw= golang.org/x/tools v0.0.0-20190130214255-bb1329dc71a0/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 h1:9oFlwfEGIvmxXTcY53ygNyxIQtWciRHjrnUvZJCYXYU= google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/store/mockstore/mocktikv/pd.go b/store/mockstore/mocktikv/pd.go index 8c78f1486724a..ac14fa9761387 100644 --- a/store/mockstore/mocktikv/pd.go +++ b/store/mockstore/mocktikv/pd.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/pd/client" ) @@ -98,7 +99,7 @@ func (c *pdClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, return store, nil } -func (c *pdClient) GetAllStores(ctx context.Context) ([]*metapb.Store, error) { +func (c *pdClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOption) ([]*metapb.Store, error) { panic(errors.New("unimplemented")) } @@ -108,3 +109,11 @@ func (c *pdClient) UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uin func (c *pdClient) Close() { } + +func (c *pdClient) ScatterRegion(ctx context.Context, regionID uint64) error { + return nil +} + +func (c *pdClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { + return nil, nil +} diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index 44d3d785c9e6f..1680f6d37eecb 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -16,6 +16,8 @@ package tikv import ( "bytes" "context" + "time" + "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" @@ -77,27 +79,12 @@ func (s *tikvStore) splitRegion(splitKey kv.Key) (*metapb.Region, error) { } } -/* func (s *tikvStore) scatterRegion(left *metapb.Region) error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - resp3, err := s.pdClient.ScatterRegion(ctx, &pdpb.ScatterRegionRequest{ - Header: &pdpb.RequestHeader{ - ClusterId: s.clusterID, - }, - RegionId: left.GetId(), - }) + err := s.pdClient.ScatterRegion(ctx, left.Id) cancel() - - if err != nil { - return err - } - if resp3.Header.GetError() != nil { - return errors.New("scatter region failed") - } - return nil - + return err } -*/ func (s *tikvStore) SplitRegionAndScatter(splitKey kv.Key) error { logutil.Logger(context.Background()).Info("split region and scatter\n\n", zap.Binary("key", splitKey)) @@ -108,7 +95,5 @@ func (s *tikvStore) SplitRegionAndScatter(splitKey kv.Key) error { if left == nil { return nil } - - return nil - + return s.scatterRegion(left) } From a88ea9b2f8c77ea40dfce8b29d282f7877a360d8 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Fri, 12 Apr 2019 16:29:19 +0800 Subject: [PATCH 03/22] fix test --- store/tikv/store_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/store/tikv/store_test.go b/store/tikv/store_test.go index d55717f7b20ac..7e296587968d2 100644 --- a/store/tikv/store_test.go +++ b/store/tikv/store_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/errors" pb "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/pd/client" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/mockoracle" @@ -168,7 +169,8 @@ func (c *mockPDClient) GetStore(ctx context.Context, storeID uint64) (*metapb.St return c.client.GetStore(ctx, storeID) } -func (c *mockPDClient) GetAllStores(ctx context.Context) ([]*metapb.Store, error) { +func (c *mockPDClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOption) ([]*metapb.Store, error) { + c.RLock() defer c.Unlock() @@ -184,6 +186,14 @@ func (c *mockPDClient) UpdateGCSafePoint(ctx context.Context, safePoint uint64) func (c *mockPDClient) Close() {} +func (c *mockPDClient) ScatterRegion(ctx context.Context, regionID uint64) error { + return nil +} + +func (c *mockPDClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { + return nil, nil +} + type checkRequestClient struct { Client priority pb.CommandPri From 3145795f3f2791d8fa90d41a52ccac33926a51db Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Fri, 12 Apr 2019 16:53:51 +0800 Subject: [PATCH 04/22] only split region for positive num --- ddl/table.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ddl/table.go b/ddl/table.go index fe88708b6e29c..9a06eafa9fd5b 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -348,7 +348,9 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo, shardBits u } // split table region step := int64(1 << (tblInfo.ShardRowIDBits - shardBits)) - max := int64(1 << tblInfo.ShardRowIDBits) + // The highest bit is the symbol bit， and alloc _tidb_rowid will always be positive number. + // So we only need to split the region for the positive number. + max := int64(1 << (tblInfo.ShardRowIDBits - 1)) for p := int64(step); p < max; p += step { recordID := p << (64 - tblInfo.ShardRowIDBits) recordPrefix := tablecodec.GenTableRecordPrefix(tblInfo.ID) From 2fccfabc9057ea243c45aa01731a57ec820d6090 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Fri, 12 Apr 2019 18:06:40 +0800 Subject: [PATCH 05/22] add PreSplitRegions option --- ddl/ddl_api.go | 5 +++++ ddl/table.go | 12 ++++++------ go.mod | 7 ++----- go.sum | 40 ++++++++++++++++------------------------ 4 files changed, 29 insertions(+), 35 deletions(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index c7c79bf614136..449cb7536470f 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -1564,8 +1564,13 @@ func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) err tbInfo.ShardRowIDBits = shardRowIDBitsMax } tbInfo.MaxShardRowIDBits = tbInfo.ShardRowIDBits + case ast.TableOptionPreSplitRegion: + tbInfo.PreSplitRegions = op.UintValue } } + if tbInfo.PreSplitRegions > tbInfo.ShardRowIDBits { + tbInfo.PreSplitRegions = tbInfo.ShardRowIDBits + } return nil } diff --git a/ddl/table.go b/ddl/table.go index 9a06eafa9fd5b..230def269dbfb 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -75,8 +75,8 @@ func onCreateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) go splitTableRegion(d.store, tbInfo.ID) } - if tbInfo.ShardRowIDBits > 0 { - go preSplitTableRegion(d.store, tbInfo, tbInfo.ShardRowIDBits/2) + if tbInfo.ShardRowIDBits > 0 && tbInfo.PreSplitRegions > 0 { + go preSplitTableRegion(d.store, tbInfo) } // Finish this job. @@ -341,13 +341,13 @@ func splitTableRegion(store kv.Storage, tableID int64) { } } -func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo, shardBits uint64) { +func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo) { s, ok := store.(splitableStore) if !ok { return } // split table region - step := int64(1 << (tblInfo.ShardRowIDBits - shardBits)) + step := int64(1 << (tblInfo.ShardRowIDBits - tblInfo.PreSplitRegions)) // The highest bit is the symbol bit， and alloc _tidb_rowid will always be positive number. // So we only need to split the region for the positive number. max := int64(1 << (tblInfo.ShardRowIDBits - 1)) @@ -356,7 +356,7 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo, shardBits u recordPrefix := tablecodec.GenTableRecordPrefix(tblInfo.ID) key := tablecodec.EncodeRecordKey(recordPrefix, recordID) if err := s.SplitRegionAndScatter(key); err != nil { - logutil.Logger(ddlLogCtx).Warn("[ddl] split table region failed", zap.Error(err)) + logutil.Logger(ddlLogCtx).Warn("[ddl] pre split table region failed", zap.Int64("recordID", recordID), zap.Error(err)) } } @@ -364,7 +364,7 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo, shardBits u for _, idx := range tblInfo.Indices { indexPrefix := tablecodec.EncodeTableIndexPrefix(tblInfo.ID, idx.ID) if err := s.SplitRegionAndScatter(indexPrefix); err != nil { - logutil.Logger(ddlLogCtx).Warn("[ddl] split table region failed", zap.Error(err)) + logutil.Logger(ddlLogCtx).Warn("[ddl] pre split table index region failed", zap.String("index", idx.Name.L), zap.Error(err)) } } } diff --git a/go.mod b/go.mod index a3ec2ca3ceda3..f72d506487c2c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 github.com/dustin/go-humanize v1.0.0 // indirect - github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a // indirect github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4 github.com/gogo/protobuf v1.2.0 // indirect @@ -26,7 +25,6 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway v1.5.1 // indirect github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 - github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 // indirect github.com/myesui/uuid v1.0.0 // indirect @@ -36,7 +34,6 @@ require ( github.com/onsi/gomega v1.4.3 // indirect github.com/opentracing/basictracer-go v1.0.0 github.com/opentracing/opentracing-go v1.0.2 - github.com/philhofer/fwd v1.0.0 // indirect github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 github.com/pingcap/errors v0.11.1 github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3 @@ -47,7 +44,6 @@ require ( github.com/pingcap/pd v2.1.0-rc.4+incompatible github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190107072121-abbec73437b7 - github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 // indirect github.com/prometheus/client_golang v0.9.0 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 // indirect @@ -60,7 +56,6 @@ require ( github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 github.com/struCoder/pidusage v0.1.2 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 - github.com/tinylib/msgp v1.1.0 // indirect github.com/twinj/uuid v1.0.0 github.com/uber-go/atomic v1.3.2 // indirect github.com/uber/jaeger-client-go v2.15.0+incompatible @@ -80,3 +75,5 @@ require ( ) replace github.com/pingcap/pd => github.com/nolouch/pd v0.0.0-20190411093849-8f8de63fdb9f + +replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190412094707-fc884a4502bc diff --git a/go.sum b/go.sum index cdd0ca7e74881..3fbbadd546323 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/crazycs520/parser v0.0.0-20190412094707-fc884a4502bc h1:dabUKvXSASe2t/+gxnob7aTLSHFH9XNCBGGsLYPe6+I= +github.com/crazycs520/parser v0.0.0-20190412094707-fc884a4502bc/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= @@ -44,13 +46,10 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= -github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a h1:QNEenQIsGDEEfFNSnN+h6hE1OwnHqTg7Dl9gEk1Cko4= -github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= @@ -75,6 +74,7 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= @@ -86,8 +86,6 @@ github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= @@ -100,18 +98,19 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe h1:W/GaMY0y69G4cFlmsC6B9sbuo2fP8OFP1ABjt4kPz+w= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -141,10 +140,9 @@ github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7l github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9 h1:KH4f4Si9XK6/IW50HtoaiLIFHGkapOM6w83za47UYik= github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= github.com/pingcap/errors v0.10.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -154,17 +152,11 @@ github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3 h1:04yuCf5NMvLU8rB2 github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= -github.com/pingcap/kvproto v0.0.0-20190215154024-7f2fc73ef562 h1:32oF1/8lVnBR2JVcCAnKPQATTOX0+ckRDFpjQk4Ngno= -github.com/pingcap/kvproto v0.0.0-20190215154024-7f2fc73ef562/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d h1:LJYJl+cBhkkTWD79n+n9Bp4agQ85SdF9YKY4zEtL9Kw= github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190409044748-a0b301443a30 h1:Cu+VJBHLUqI0TFj/0Kya4L1iHIJZ3VbtZcEwv+3zOxQ= -github.com/pingcap/parser v0.0.0-20190409044748-a0b301443a30/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= -github.com/pingcap/pd v2.1.0-rc.4+incompatible h1:/buwGk04aHO5odk/+O8ZOXGs4qkUjYTJ2UpCJXna8NE= -github.com/pingcap/pd v2.1.0-rc.4+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190107072121-abbec73437b7 h1:wnjdQRhybddDesBVBKyOLUPgDaOFdtqA92pduBgWvVQ= @@ -174,8 +166,6 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 h1:gGBSHPOU7g8YjTbhwn+lvFm2VDEhhA+PwDIlstkgSxE= -github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.0 h1:tXuTFVHC03mW0D+Ua1Q2d1EAVqLTuggX50V0VLICCzY= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -205,6 +195,7 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -213,11 +204,10 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/struCoder/pidusage v0.1.2 h1:fFPTThlcWFQyizv3xKs5Lyq1lpG5lZ36arEGNhWz2Vs= github.com/struCoder/pidusage v0.1.2/go.mod h1:pWBlW3YuSwRl6h7R5KbvA4N8oOqe9LjaKW5CwT1SPjI= +github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc= github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= -github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -229,19 +219,20 @@ github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1 github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v0.0.0-20171019201919-bdcc60b419d1 h1:UvhxfNjNqlZ/x3cDyqxMhoiUpemd3zXkVQApN6bM/lg= -github.com/ugorji/go v0.0.0-20171019201919-bdcc60b419d1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go v1.1.2 h1:JON3E2/GPW2iDNGoSAusl1KDf5TRQ8k8q7Tp097pZGs= github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 h1:BasDe+IErOQKrMVXab7UayvSlIpiyGwRvuX3EKYY7UA= github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d h1:ggUgChAeyge4NZ4QUw6lhHsVymzwSDJOZcE0s2X8S20= github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= -github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs= -github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20190320044326-77d4b742cdbf h1:rmttwKPEgG/l4UscTDYtaJgeUsedKPKSyFfNQLI6q+I= go.etcd.io/etcd v0.0.0-20190320044326-77d4b742cdbf/go.mod h1:KSGwdbiFchh5KIC9My2+ZVl5/3ANcwohw50dpPwa2cw= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -291,6 +282,7 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= From 77f0a96b98dca93c87d6eed324226d0e8f5d406f Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Fri, 12 Apr 2019 19:48:27 +0800 Subject: [PATCH 06/22] add retry for scatter region --- store/tikv/backoff.go | 1 + store/tikv/split_region.go | 29 +++++++++++++++++++---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/store/tikv/backoff.go b/store/tikv/backoff.go index 8d139903bfb32..bc65427308c43 100644 --- a/store/tikv/backoff.go +++ b/store/tikv/backoff.go @@ -173,6 +173,7 @@ const ( deleteRangeOneRegionMaxBackoff = 100000 rawkvMaxBackoff = 20000 splitRegionBackoff = 20000 + scatterRegionBackoff = 20000 ) // CommitMaxBackoff is max sleep time of the 'commit' command diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index 1680f6d37eecb..74b4db4648732 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -16,8 +16,6 @@ package tikv import ( "bytes" "context" - "time" - "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" @@ -80,20 +78,31 @@ func (s *tikvStore) splitRegion(splitKey kv.Key) (*metapb.Region, error) { } func (s *tikvStore) scatterRegion(left *metapb.Region) error { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - err := s.pdClient.ScatterRegion(ctx, left.Id) - cancel() - return err + if left == nil { + return nil + } + logutil.Logger(context.Background()).Info("start scatter region", + zap.Uint64("regionID", left.Id)) + bo := NewBackoffer(context.Background(), scatterRegionBackoff) + for { + err := s.pdClient.ScatterRegion(context.Background(), left.Id) + if err != nil { + err = bo.Backoff(BoRegionMiss, errors.New(err.Error())) + if err != nil { + return errors.Trace(err) + } + continue + } + logutil.Logger(context.Background()).Info("scatter region complete", + zap.Uint64("regionID", left.Id)) + return nil + } } func (s *tikvStore) SplitRegionAndScatter(splitKey kv.Key) error { - logutil.Logger(context.Background()).Info("split region and scatter\n\n", zap.Binary("key", splitKey)) left, err := s.splitRegion(splitKey) if err != nil { return err } - if left == nil { - return nil - } return s.scatterRegion(left) } From 189335dbf0519a7b83c26d02e431913a1d69b702 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Fri, 12 Apr 2019 20:26:16 +0800 Subject: [PATCH 07/22] refine comment --- ddl/table.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddl/table.go b/ddl/table.go index 230def269dbfb..665a2951afee6 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -346,7 +346,7 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo) { if !ok { return } - // split table region + // Split table region. step := int64(1 << (tblInfo.ShardRowIDBits - tblInfo.PreSplitRegions)) // The highest bit is the symbol bit， and alloc _tidb_rowid will always be positive number. // So we only need to split the region for the positive number. @@ -360,7 +360,7 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo) { } } - // split index region. + // Split index region. for _, idx := range tblInfo.Indices { indexPrefix := tablecodec.EncodeTableIndexPrefix(tblInfo.ID, idx.ID) if err := s.SplitRegionAndScatter(indexPrefix); err != nil { From b517f03291a1864ffa7fe9da8045ca20918583c9 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Mon, 15 Apr 2019 10:31:25 +0800 Subject: [PATCH 08/22] remove blank line --- store/tikv/store_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/store/tikv/store_test.go b/store/tikv/store_test.go index 7e296587968d2..93b725f0af84f 100644 --- a/store/tikv/store_test.go +++ b/store/tikv/store_test.go @@ -170,7 +170,6 @@ func (c *mockPDClient) GetStore(ctx context.Context, storeID uint64) (*metapb.St } func (c *mockPDClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOption) ([]*metapb.Store, error) { - c.RLock() defer c.Unlock() From 55cd7be36fea16f29c67001de4f550a337e72f20 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 16 Apr 2019 17:02:15 +0800 Subject: [PATCH 09/22] update go.mod for pd --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index f72d506487c2c..ada7f128ba669 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 github.com/pingcap/parser v0.0.0-20190409044748-a0b301443a30 - github.com/pingcap/pd v2.1.0-rc.4+incompatible + github.com/pingcap/pd v0.0.0-20190415104203-fc997de0ae38 github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190107072121-abbec73437b7 github.com/prometheus/client_golang v0.9.0 @@ -74,6 +74,4 @@ require ( sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) -replace github.com/pingcap/pd => github.com/nolouch/pd v0.0.0-20190411093849-8f8de63fdb9f - replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190412094707-fc884a4502bc diff --git a/go.sum b/go.sum index 3fbbadd546323..4a37bd3fa059c 100644 --- a/go.sum +++ b/go.sum @@ -127,8 +127,6 @@ github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdc github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= -github.com/nolouch/pd v0.0.0-20190411093849-8f8de63fdb9f h1:ys7t3nE/ycGCknGGctRh4l2r2KloucokCxyjwOLrkxQ= -github.com/nolouch/pd v0.0.0-20190411093849-8f8de63fdb9f/go.mod h1:MUCxRzOkYiWZtlyi4MhxjCIj9PgQQ/j+BLNGm7aUsnM= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= @@ -157,6 +155,8 @@ github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= +github.com/pingcap/pd v0.0.0-20190415104203-fc997de0ae38 h1:O4lVW+nYoMrJBrf04VJ5wAdRQxX2bg1ZT86InMgVaVA= +github.com/pingcap/pd v0.0.0-20190415104203-fc997de0ae38/go.mod h1:MUCxRzOkYiWZtlyi4MhxjCIj9PgQQ/j+BLNGm7aUsnM= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190107072121-abbec73437b7 h1:wnjdQRhybddDesBVBKyOLUPgDaOFdtqA92pduBgWvVQ= From eb27dfbdc543374f4ddd8b9effc4ff11b83a2b7a Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 17 Apr 2019 17:16:46 +0800 Subject: [PATCH 10/22] add wait scatter finish --- ddl/table.go | 8 ++--- store/mockstore/mocktikv/pd.go | 2 +- store/tikv/split_region.go | 54 +++++++++++++++++++++++++++------- store/tikv/store_test.go | 2 +- 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/ddl/table.go b/ddl/table.go index 665a2951afee6..462715ba97d9e 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -76,7 +76,7 @@ func onCreateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) } if tbInfo.ShardRowIDBits > 0 && tbInfo.PreSplitRegions > 0 { - go preSplitTableRegion(d.store, tbInfo) + preSplitTableRegion(d.store, tbInfo) } // Finish this job. @@ -326,7 +326,7 @@ func checkSafePoint(w *worker, snapshotTS uint64) error { type splitableStore interface { SplitRegion(splitKey kv.Key) error - SplitRegionAndScatter(splitKey kv.Key) error + SplitRegionAndScatter(splitKey kv.Key, waitFinish bool) error } func splitTableRegion(store kv.Storage, tableID int64) { @@ -355,7 +355,7 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo) { recordID := p << (64 - tblInfo.ShardRowIDBits) recordPrefix := tablecodec.GenTableRecordPrefix(tblInfo.ID) key := tablecodec.EncodeRecordKey(recordPrefix, recordID) - if err := s.SplitRegionAndScatter(key); err != nil { + if err := s.SplitRegionAndScatter(key, true); err != nil { logutil.Logger(ddlLogCtx).Warn("[ddl] pre split table region failed", zap.Int64("recordID", recordID), zap.Error(err)) } } @@ -363,7 +363,7 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo) { // Split index region. for _, idx := range tblInfo.Indices { indexPrefix := tablecodec.EncodeTableIndexPrefix(tblInfo.ID, idx.ID) - if err := s.SplitRegionAndScatter(indexPrefix); err != nil { + if err := s.SplitRegionAndScatter(indexPrefix, true); err != nil { logutil.Logger(ddlLogCtx).Warn("[ddl] pre split table index region failed", zap.String("index", idx.Name.L), zap.Error(err)) } } diff --git a/store/mockstore/mocktikv/pd.go b/store/mockstore/mocktikv/pd.go index ac14fa9761387..31597b7417820 100644 --- a/store/mockstore/mocktikv/pd.go +++ b/store/mockstore/mocktikv/pd.go @@ -115,5 +115,5 @@ func (c *pdClient) ScatterRegion(ctx context.Context, regionID uint64) error { } func (c *pdClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { - return nil, nil + return &pdpb.GetOperatorResponse{Status: pdpb.OperatorStatus_SUCCESS}, nil } diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index 74b4db4648732..46d72fb3039b0 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/util/logutil" @@ -77,15 +78,12 @@ func (s *tikvStore) splitRegion(splitKey kv.Key) (*metapb.Region, error) { } } -func (s *tikvStore) scatterRegion(left *metapb.Region) error { - if left == nil { - return nil - } +func (s *tikvStore) scatterRegion(regionID uint64, waitFinish bool) error { logutil.Logger(context.Background()).Info("start scatter region", - zap.Uint64("regionID", left.Id)) + zap.Uint64("regionID", regionID)) bo := NewBackoffer(context.Background(), scatterRegionBackoff) for { - err := s.pdClient.ScatterRegion(context.Background(), left.Id) + err := s.pdClient.ScatterRegion(context.Background(), regionID) if err != nil { err = bo.Backoff(BoRegionMiss, errors.New(err.Error())) if err != nil { @@ -93,16 +91,52 @@ func (s *tikvStore) scatterRegion(left *metapb.Region) error { } continue } - logutil.Logger(context.Background()).Info("scatter region complete", - zap.Uint64("regionID", left.Id)) + break + } + logutil.Logger(context.Background()).Info("scatter region complete", + zap.Uint64("regionID", regionID)) + if !waitFinish { return nil } + return s.waitScatterRegionFinish(regionID) +} + +func (s *tikvStore) waitScatterRegionFinish(regionID uint64) error { + logutil.Logger(context.Background()).Info("wait scatter region", + zap.Uint64("regionID", regionID)) + bo := NewBackoffer(context.Background(), scatterRegionBackoff) + for { + resp, err := s.pdClient.GetOperator(context.Background(), regionID) + if err == nil && resp != nil { + if !bytes.Equal(resp.Desc, []byte("scatter-region")) || resp.Status != pdpb.OperatorStatus_RUNNING { + logutil.Logger(context.Background()).Info("wait scatter region finished", + zap.Uint64("regionID", regionID)) + return nil + } + logutil.Logger(context.Background()).Info("wait scatter region", + zap.Uint64("regionID", regionID), + zap.String("desc", string(resp.Desc)), + zap.String("status", pdpb.OperatorStatus_name[int32(resp.Status)])) + } + if err != nil { + err = bo.Backoff(BoRegionMiss, errors.New(err.Error())) + } else { + err = bo.Backoff(BoRegionMiss, errors.New("wait scatter region")) + } + if err != nil { + return errors.Trace(err) + } + } + } -func (s *tikvStore) SplitRegionAndScatter(splitKey kv.Key) error { +func (s *tikvStore) SplitRegionAndScatter(splitKey kv.Key, waitFinish bool) error { left, err := s.splitRegion(splitKey) if err != nil { return err } - return s.scatterRegion(left) + if left == nil { + return nil + } + return s.scatterRegion(left.Id, waitFinish) } diff --git a/store/tikv/store_test.go b/store/tikv/store_test.go index 93b725f0af84f..66bd99c286e43 100644 --- a/store/tikv/store_test.go +++ b/store/tikv/store_test.go @@ -190,7 +190,7 @@ func (c *mockPDClient) ScatterRegion(ctx context.Context, regionID uint64) error } func (c *mockPDClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { - return nil, nil + return &pdpb.GetOperatorResponse{Status: pdpb.OperatorStatus_SUCCESS}, nil } type checkRequestClient struct { From a70696f6d00d25693e8bbc7cc49a0d3101184ea7 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 17 Apr 2019 18:05:29 +0800 Subject: [PATCH 11/22] add table option: wait_split_finish --- ddl/ddl_api.go | 10 ++++++++++ ddl/table.go | 8 ++------ go.mod | 2 +- go.sum | 2 ++ 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 6d8dc26200375..0d5edb6a33d7f 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -1197,6 +1197,14 @@ func (d *ddl) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) (err e err = d.doDDLJob(ctx, job) if err == nil { + // do pre-split and scatter. + if tbInfo.ShardRowIDBits > 0 && tbInfo.PreSplitRegions > 0 { + if tbInfo.WaitSplitFinish { + preSplitTableRegion(d.store, tbInfo) + } else { + go preSplitTableRegion(d.store, tbInfo) + } + } if tbInfo.AutoIncID > 1 { // Default tableAutoIncID base is 0. // If the first ID is expected to greater than 1, we need to do rebase. @@ -1571,6 +1579,8 @@ func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) err tbInfo.MaxShardRowIDBits = tbInfo.ShardRowIDBits case ast.TableOptionPreSplitRegion: tbInfo.PreSplitRegions = op.UintValue + case ast.TableOptionWaitSplitFinish: + tbInfo.WaitSplitFinish = op.UintValue == 1 } } if tbInfo.PreSplitRegions > tbInfo.ShardRowIDBits { diff --git a/ddl/table.go b/ddl/table.go index 462715ba97d9e..e1805820b7e5b 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -75,10 +75,6 @@ func onCreateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) go splitTableRegion(d.store, tbInfo.ID) } - if tbInfo.ShardRowIDBits > 0 && tbInfo.PreSplitRegions > 0 { - preSplitTableRegion(d.store, tbInfo) - } - // Finish this job. job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo) asyncNotifyEvent(d, &util.Event{Tp: model.ActionCreateTable, TableInfo: tbInfo}) @@ -355,7 +351,7 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo) { recordID := p << (64 - tblInfo.ShardRowIDBits) recordPrefix := tablecodec.GenTableRecordPrefix(tblInfo.ID) key := tablecodec.EncodeRecordKey(recordPrefix, recordID) - if err := s.SplitRegionAndScatter(key, true); err != nil { + if err := s.SplitRegionAndScatter(key, tblInfo.WaitSplitFinish); err != nil { logutil.Logger(ddlLogCtx).Warn("[ddl] pre split table region failed", zap.Int64("recordID", recordID), zap.Error(err)) } } @@ -363,7 +359,7 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo) { // Split index region. for _, idx := range tblInfo.Indices { indexPrefix := tablecodec.EncodeTableIndexPrefix(tblInfo.ID, idx.ID) - if err := s.SplitRegionAndScatter(indexPrefix, true); err != nil { + if err := s.SplitRegionAndScatter(indexPrefix, tblInfo.WaitSplitFinish); err != nil { logutil.Logger(ddlLogCtx).Warn("[ddl] pre split table index region failed", zap.String("index", idx.Name.L), zap.Error(err)) } } diff --git a/go.mod b/go.mod index ada7f128ba669..b7c3ade9ae7d8 100644 --- a/go.mod +++ b/go.mod @@ -74,4 +74,4 @@ require ( sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) -replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190412094707-fc884a4502bc +replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190417094927-cdb39be36472 diff --git a/go.sum b/go.sum index 4a37bd3fa059c..2eda6b66f71fe 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/crazycs520/parser v0.0.0-20190412094707-fc884a4502bc h1:dabUKvXSASe2t/+gxnob7aTLSHFH9XNCBGGsLYPe6+I= github.com/crazycs520/parser v0.0.0-20190412094707-fc884a4502bc/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= +github.com/crazycs520/parser v0.0.0-20190417094927-cdb39be36472 h1:C4cE3dwbZ+U7+eJgMFpShhuzl/qR6qvCtqCXsVuIyaE= +github.com/crazycs520/parser v0.0.0-20190417094927-cdb39be36472/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= From e362d68193311ae9b14659bbd8bda7e4938d3076 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Thu, 18 Apr 2019 17:17:24 +0800 Subject: [PATCH 12/22] split and scatter region first, wait scatter finish together --- ddl/table.go | 23 ++++++++++++++++++++--- store/tikv/backoff.go | 1 + store/tikv/split_region.go | 37 +++++++++++++++++++++---------------- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/ddl/table.go b/ddl/table.go index e1805820b7e5b..04e7732407d2e 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -322,7 +322,8 @@ func checkSafePoint(w *worker, snapshotTS uint64) error { type splitableStore interface { SplitRegion(splitKey kv.Key) error - SplitRegionAndScatter(splitKey kv.Key, waitFinish bool) error + SplitRegionAndScatter(splitKey kv.Key) (uint64, error) + WaitScatterRegionFinish(regionID uint64) error } func splitTableRegion(store kv.Storage, tableID int64) { @@ -342,6 +343,7 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo) { if !ok { return } + regionIDs := make([]uint64, 0, 1<<(tblInfo.PreSplitRegions-1)+len(tblInfo.Indices)) // Split table region. step := int64(1 << (tblInfo.ShardRowIDBits - tblInfo.PreSplitRegions)) // The highest bit is the symbol bit， and alloc _tidb_rowid will always be positive number. @@ -351,16 +353,31 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo) { recordID := p << (64 - tblInfo.ShardRowIDBits) recordPrefix := tablecodec.GenTableRecordPrefix(tblInfo.ID) key := tablecodec.EncodeRecordKey(recordPrefix, recordID) - if err := s.SplitRegionAndScatter(key, tblInfo.WaitSplitFinish); err != nil { + regionID, err := s.SplitRegionAndScatter(key) + if err != nil { logutil.Logger(ddlLogCtx).Warn("[ddl] pre split table region failed", zap.Int64("recordID", recordID), zap.Error(err)) + } else { + regionIDs = append(regionIDs, regionID) } } // Split index region. for _, idx := range tblInfo.Indices { indexPrefix := tablecodec.EncodeTableIndexPrefix(tblInfo.ID, idx.ID) - if err := s.SplitRegionAndScatter(indexPrefix, tblInfo.WaitSplitFinish); err != nil { + regionID, err := s.SplitRegionAndScatter(indexPrefix) + if err != nil { logutil.Logger(ddlLogCtx).Warn("[ddl] pre split table index region failed", zap.String("index", idx.Name.L), zap.Error(err)) + } else { + regionIDs = append(regionIDs, regionID) + } + } + if !tblInfo.WaitSplitFinish { + return + } + for _, regionID := range regionIDs { + err := s.WaitScatterRegionFinish(regionID) + if err != nil { + logutil.Logger(ddlLogCtx).Warn("[ddl] wait scatter region failed", zap.Uint64("regionID", regionID), zap.Error(err)) } } } diff --git a/store/tikv/backoff.go b/store/tikv/backoff.go index bc65427308c43..7a29eeec5cd8b 100644 --- a/store/tikv/backoff.go +++ b/store/tikv/backoff.go @@ -174,6 +174,7 @@ const ( rawkvMaxBackoff = 20000 splitRegionBackoff = 20000 scatterRegionBackoff = 20000 + waitScatterRegionFinishBackoff = 60000 ) // CommitMaxBackoff is max sleep time of the 'commit' command diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index 46d72fb3039b0..046d57b4469cc 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -78,7 +78,7 @@ func (s *tikvStore) splitRegion(splitKey kv.Key) (*metapb.Region, error) { } } -func (s *tikvStore) scatterRegion(regionID uint64, waitFinish bool) error { +func (s *tikvStore) scatterRegion(regionID uint64) error { logutil.Logger(context.Background()).Info("start scatter region", zap.Uint64("regionID", regionID)) bo := NewBackoffer(context.Background(), scatterRegionBackoff) @@ -95,16 +95,14 @@ func (s *tikvStore) scatterRegion(regionID uint64, waitFinish bool) error { } logutil.Logger(context.Background()).Info("scatter region complete", zap.Uint64("regionID", regionID)) - if !waitFinish { - return nil - } - return s.waitScatterRegionFinish(regionID) + return nil } -func (s *tikvStore) waitScatterRegionFinish(regionID uint64) error { +func (s *tikvStore) WaitScatterRegionFinish(regionID uint64) error { logutil.Logger(context.Background()).Info("wait scatter region", zap.Uint64("regionID", regionID)) - bo := NewBackoffer(context.Background(), scatterRegionBackoff) + bo := NewBackoffer(context.Background(), waitScatterRegionFinishBackoff) + logFreq := 0 for { resp, err := s.pdClient.GetOperator(context.Background(), regionID) if err == nil && resp != nil { @@ -113,15 +111,18 @@ func (s *tikvStore) waitScatterRegionFinish(regionID uint64) error { zap.Uint64("regionID", regionID)) return nil } - logutil.Logger(context.Background()).Info("wait scatter region", - zap.Uint64("regionID", regionID), - zap.String("desc", string(resp.Desc)), - zap.String("status", pdpb.OperatorStatus_name[int32(resp.Status)])) + if logFreq%10 == 0 { + logutil.Logger(context.Background()).Info("wait scatter region", + zap.Uint64("regionID", regionID), + zap.String("desc", string(resp.Desc)), + zap.String("status", pdpb.OperatorStatus_name[int32(resp.Status)])) + } + logFreq++ } if err != nil { err = bo.Backoff(BoRegionMiss, errors.New(err.Error())) } else { - err = bo.Backoff(BoRegionMiss, errors.New("wait scatter region")) + err = bo.Backoff(BoRegionMiss, errors.New("wait scatter region timeout")) } if err != nil { return errors.Trace(err) @@ -130,13 +131,17 @@ func (s *tikvStore) waitScatterRegionFinish(regionID uint64) error { } -func (s *tikvStore) SplitRegionAndScatter(splitKey kv.Key, waitFinish bool) error { +func (s *tikvStore) SplitRegionAndScatter(splitKey kv.Key) (uint64, error) { left, err := s.splitRegion(splitKey) if err != nil { - return err + return 0, err } if left == nil { - return nil + return 0, nil + } + err = s.scatterRegion(left.Id) + if err != nil { + return 0, err } - return s.scatterRegion(left.Id, waitFinish) + return left.Id, nil } From 918b183b014298e50c6285695a8040623db25520 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Fri, 19 Apr 2019 22:37:03 +0800 Subject: [PATCH 13/22] *: add split table index syntax --- ddl/ddl_api.go | 8 +--- ddl/table.go | 6 +-- executor/builder.go | 13 +++++++ executor/split_index.go | 71 ++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 6 +-- planner/core/common_plans.go | 9 +++++ planner/core/planbuilder.go | 50 +++++++++++++++++++++++++ 8 files changed, 150 insertions(+), 15 deletions(-) create mode 100644 executor/split_index.go diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 0d5edb6a33d7f..a2e509f3457e2 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -1199,11 +1199,7 @@ func (d *ddl) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) (err e if err == nil { // do pre-split and scatter. if tbInfo.ShardRowIDBits > 0 && tbInfo.PreSplitRegions > 0 { - if tbInfo.WaitSplitFinish { - preSplitTableRegion(d.store, tbInfo) - } else { - go preSplitTableRegion(d.store, tbInfo) - } + go preSplitTableRegion(d.store, tbInfo) } if tbInfo.AutoIncID > 1 { // Default tableAutoIncID base is 0. @@ -1579,8 +1575,6 @@ func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) err tbInfo.MaxShardRowIDBits = tbInfo.ShardRowIDBits case ast.TableOptionPreSplitRegion: tbInfo.PreSplitRegions = op.UintValue - case ast.TableOptionWaitSplitFinish: - tbInfo.WaitSplitFinish = op.UintValue == 1 } } if tbInfo.PreSplitRegions > tbInfo.ShardRowIDBits { diff --git a/ddl/table.go b/ddl/table.go index 04e7732407d2e..4532637fd5e29 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -371,9 +371,9 @@ func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo) { regionIDs = append(regionIDs, regionID) } } - if !tblInfo.WaitSplitFinish { - return - } + //if !tblInfo.WaitSplitFinish { + // return + //} for _, regionID := range regionIDs { err := s.WaitScatterRegionFinish(regionID) if err != nil { diff --git a/executor/builder.go b/executor/builder.go index 6ccddeb8ca22c..dc0a98991bc8b 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -180,6 +180,8 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor { return b.buildWindow(v) case *plannercore.SQLBindPlan: return b.buildSQLBindExec(v) + case *plannercore.SplitIndexRegion: + return b.buildSplitIndexRegion(v) default: if mp, ok := p.(MockPhysicalPlan); ok { return mp.GetExecutor() @@ -1229,6 +1231,17 @@ func (b *executorBuilder) buildUnionAll(v *plannercore.PhysicalUnionAll) Executo return e } +func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitIndexRegion) Executor { + base := newBaseExecutor(b.ctx, nil, v.ExplainID()) + base.initCap = chunk.ZeroCapacity + return &SplitIndexRegionExec{ + baseExecutor: base, + table: v.Table, + indexInfo: v.IndexInfo, + valueLists: v.ValueLists, + } +} + func (b *executorBuilder) buildUpdate(v *plannercore.Update) Executor { tblID2table := make(map[int64]table.Table) for id := range v.SelectPlan.Schema().TblID2Handle { diff --git a/executor/split_index.go b/executor/split_index.go new file mode 100644 index 0000000000000..b181589515e4a --- /dev/null +++ b/executor/split_index.go @@ -0,0 +1,71 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "context" + + "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/table/tables" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/logutil" + "go.uber.org/zap" +) + +type SplitIndexRegionExec struct { + baseExecutor + + table table.Table + indexInfo *model.IndexInfo + valueLists [][]types.Datum +} + +type splitableStore interface { + SplitRegionAndScatter(splitKey kv.Key) (uint64, error) + WaitScatterRegionFinish(regionID uint64) error +} + +func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) error { + store := e.ctx.GetStore() + s, ok := store.(splitableStore) + if !ok { + return nil + } + regionIDs := make([]uint64, 0, len(e.valueLists)) + index := tables.NewIndex(e.table.Meta().ID, e.table.Meta(), e.indexInfo) + for _, values := range e.valueLists { + idxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, values, 1, nil) + if err != nil { + return err + } + + regionID, err := s.SplitRegionAndScatter(idxKey) + if err != nil { + logutil.Logger(context.Background()).Warn("split table index region failed", zap.String("table", e.table.Meta().Name.L), zap.String("index", e.indexInfo.Name.L), zap.Error(err)) + continue + } + regionIDs = append(regionIDs, regionID) + + } + for _, regionID := range regionIDs { + err := s.WaitScatterRegionFinish(regionID) + if err != nil { + logutil.Logger(context.Background()).Warn("wait scatter region failed", zap.Uint64("regionID", regionID), zap.Error(err)) + } + } + return nil +} diff --git a/go.mod b/go.mod index b7c3ade9ae7d8..760652cd868ad 100644 --- a/go.mod +++ b/go.mod @@ -74,4 +74,4 @@ require ( sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) -replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190417094927-cdb39be36472 +replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190419130829-155f41ba841e diff --git a/go.sum b/go.sum index 2eda6b66f71fe..11559ffe7850e 100644 --- a/go.sum +++ b/go.sum @@ -29,10 +29,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/crazycs520/parser v0.0.0-20190412094707-fc884a4502bc h1:dabUKvXSASe2t/+gxnob7aTLSHFH9XNCBGGsLYPe6+I= -github.com/crazycs520/parser v0.0.0-20190412094707-fc884a4502bc/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= -github.com/crazycs520/parser v0.0.0-20190417094927-cdb39be36472 h1:C4cE3dwbZ+U7+eJgMFpShhuzl/qR6qvCtqCXsVuIyaE= -github.com/crazycs520/parser v0.0.0-20190417094927-cdb39be36472/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= +github.com/crazycs520/parser v0.0.0-20190419130829-155f41ba841e h1:kWKC232mkwfm2thmpWIFG4uIPqispxjcoog/qIjzbsg= +github.com/crazycs520/parser v0.0.0-20190419130829-155f41ba841e/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 15448100a4d61..18848236834b1 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/types" driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/kvcache" @@ -459,6 +460,14 @@ type LoadStats struct { Path string } +type SplitIndexRegion struct { + baseSchemaProducer + + Table table.Table + IndexInfo *model.IndexInfo + ValueLists [][]types.Datum +} + // DDL represents a DDL statement plan. type DDL struct { baseSchemaProducer diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index ab8721a9f343e..d105ac2182553 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -277,6 +277,8 @@ func (b *PlanBuilder) Build(node ast.Node) (Plan, error) { return b.buildCreateBindPlan(x) case *ast.ChangeStmt: return b.buildChange(x) + case *ast.SplitIndexRegionStmt: + return b.buildSplitIndexRegion(x) } return nil, ErrUnsupportedType.GenWithStack("Unsupported type %T", node) } @@ -1573,6 +1575,54 @@ func (b *PlanBuilder) buildLoadStats(ld *ast.LoadStatsStmt) Plan { return p } +func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Plan, error) { + tblInfo := node.Table.TableInfo + indexInfo := tblInfo.FindIndexByName(strings.ToLower(node.IndexName)) + if indexInfo == nil { + return nil, ErrKeyDoesNotExist.GenWithStackByArgs(node.IndexName, tblInfo.Name) + } + schema := expression.TableInfo2SchemaWithDBName(b.ctx, node.Table.Schema, tblInfo) + mockTablePlan := LogicalTableDual{}.Init(b.ctx) + mockTablePlan.SetSchema(schema) + + tps := make([]*types.FieldType, 0, len(indexInfo.Columns)) + for _, c := range indexInfo.Columns { + col := tblInfo.Columns[c.Offset] + tps = append(tps, &col.FieldType) + } + + indexValues := make([][]types.Datum, 0, len(node.ValueLists)) + for i, valuesItem := range node.ValueLists { + if len(valuesItem) > len(indexInfo.Columns) { + return nil, ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1) + } + valueList := make([]types.Datum, 0, len(valuesItem)) + for j, valueItem := range valuesItem { + x, ok := valueItem.(*driver.ValueExpr) + if !ok { + return nil, errors.Trace(errors.New("expect constant values")) + } + value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, tps[j]) + if err != nil { + return nil, err + } + + valueList = append(valueList, value) + } + indexValues = append(indexValues, valueList) + } + tableInPlan, ok := b.is.TableByID(tblInfo.ID) + if !ok { + return nil, errors.Errorf("Can't get table %s.", tblInfo.Name.O) + } + return &SplitIndexRegion{ + Table: tableInPlan, + IndexInfo: indexInfo, + ValueLists: indexValues, + }, nil + +} + func (b *PlanBuilder) buildDDL(node ast.DDLNode) (Plan, error) { var authErr error switch v := node.(type) { From 87ec41a2fb3e014939a6a77019c532c5b35765f4 Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 22 Apr 2019 10:02:48 +0800 Subject: [PATCH 14/22] wait scatter finish when create table --- ddl/ddl_api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index a2e509f3457e2..e03700b33e8e2 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -1199,7 +1199,7 @@ func (d *ddl) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) (err e if err == nil { // do pre-split and scatter. if tbInfo.ShardRowIDBits > 0 && tbInfo.PreSplitRegions > 0 { - go preSplitTableRegion(d.store, tbInfo) + preSplitTableRegion(d.store, tbInfo) } if tbInfo.AutoIncID > 1 { // Default tableAutoIncID base is 0. From 6cfd51ec68bcddfe2e532da5cecc01a9b477daea Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 25 Apr 2019 09:31:17 +0800 Subject: [PATCH 15/22] refine code --- ddl/table.go | 1 - executor/{split_index.go => split.go} | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) rename executor/{split_index.go => split.go} (90%) diff --git a/ddl/table.go b/ddl/table.go index 34f41c425b058..225a11208a62b 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -78,7 +78,6 @@ func onCreateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) // TODO: Add restrictions to this operation. go splitTableRegion(d.store, tbInfo.ID) } - // Finish this job. job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo) asyncNotifyEvent(d, &util.Event{Tp: model.ActionCreateTable, TableInfo: tbInfo}) diff --git a/executor/split_index.go b/executor/split.go similarity index 90% rename from executor/split_index.go rename to executor/split.go index b181589515e4a..3a3d6d961e198 100644 --- a/executor/split_index.go +++ b/executor/split.go @@ -55,7 +55,10 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) e regionID, err := s.SplitRegionAndScatter(idxKey) if err != nil { - logutil.Logger(context.Background()).Warn("split table index region failed", zap.String("table", e.table.Meta().Name.L), zap.String("index", e.indexInfo.Name.L), zap.Error(err)) + logutil.Logger(context.Background()).Warn("split table index region failed", + zap.String("table", e.table.Meta().Name.L), + zap.String("index", e.indexInfo.Name.L), + zap.Error(err)) continue } regionIDs = append(regionIDs, regionID) @@ -64,7 +67,9 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) e for _, regionID := range regionIDs { err := s.WaitScatterRegionFinish(regionID) if err != nil { - logutil.Logger(context.Background()).Warn("wait scatter region failed", zap.Uint64("regionID", regionID), zap.Error(err)) + logutil.Logger(context.Background()).Warn("wait scatter region failed", + zap.Uint64("regionID", regionID), + zap.Error(err)) } } return nil From 952b870e2b4ee114cf57d1548f3d7df1c6677d0c Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 25 Apr 2019 09:44:25 +0800 Subject: [PATCH 16/22] add comment --- executor/split.go | 2 ++ planner/core/common_plans.go | 1 + 2 files changed, 3 insertions(+) diff --git a/executor/split.go b/executor/split.go index 3a3d6d961e198..93a7fe5d830af 100644 --- a/executor/split.go +++ b/executor/split.go @@ -26,6 +26,7 @@ import ( "go.uber.org/zap" ) +// SplitIndexRegionExec represents a split index regions executor. type SplitIndexRegionExec struct { baseExecutor @@ -39,6 +40,7 @@ type splitableStore interface { WaitScatterRegionFinish(regionID uint64) error } +// Next implements the Executor Next interface. func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) error { store := e.ctx.GetStore() s, ok := store.(splitableStore) diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index ec9aa2637e102..bdbada74f698e 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -476,6 +476,7 @@ type LoadStats struct { Path string } +// SplitIndexRegion represents a split index regions plan. type SplitIndexRegion struct { baseSchemaProducer From 38a057d7c9347516787e7cd9c2cd8c2d6782f355 Mon Sep 17 00:00:00 2001 From: crazycs Date: Fri, 26 Apr 2019 22:01:36 +0800 Subject: [PATCH 17/22] address comment and add test for convert value should not ignore truncate err --- executor/executor.go | 4 ++++ executor/executor_test.go | 13 +++++++++++++ planner/core/planbuilder.go | 8 +------- store/tikv/split_region.go | 1 + 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/executor/executor.go b/executor/executor.go index cc0da1bdddf46..a9f6e35cf8e16 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1379,6 +1379,10 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { sc.InShowWarning = true sc.SetWarnings(vars.StmtCtx.GetWarnings()) } + case *ast.SplitIndexRegionStmt: + sc.IgnoreTruncate = false + sc.IgnoreZeroInDate = true + sc.AllowInvalidDate = vars.SQLMode.HasAllowInvalidDatesMode() default: sc.IgnoreTruncate = true sc.IgnoreZeroInDate = true diff --git a/executor/executor_test.go b/executor/executor_test.go index 60707de717b93..0856d15158683 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3655,6 +3655,19 @@ func (s *testSuite) TestReadPartitionedTable(c *C) { tk.MustQuery("select a from pt where b = 3").Check(testkit.Rows("3")) } +func (s *testSuite) TestSplitIndexRegion(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a varchar(100),b int, index idx1(b,a))") + tk.MustExec(`split table t index idx1 by (10000,"abcd"),(10000000);`) + _, err := tk.Exec(`split table t index idx1 by ("abcd");`) + c.Assert(err, NotNil) + terr := errors.Cause(err).(*terror.Error) + c.Assert(terr.Code(), Equals, terror.ErrCode(mysql.WarnDataTruncated)) + +} + type testOOMSuite struct { store kv.Storage do *domain.Domain diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 137869212ca75..f485ad5984638 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1600,12 +1600,6 @@ func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Pla mockTablePlan := LogicalTableDual{}.Init(b.ctx) mockTablePlan.SetSchema(schema) - tps := make([]*types.FieldType, 0, len(indexInfo.Columns)) - for _, c := range indexInfo.Columns { - col := tblInfo.Columns[c.Offset] - tps = append(tps, &col.FieldType) - } - indexValues := make([][]types.Datum, 0, len(node.ValueLists)) for i, valuesItem := range node.ValueLists { if len(valuesItem) > len(indexInfo.Columns) { @@ -1617,7 +1611,7 @@ func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Pla if !ok { return nil, errors.Trace(errors.New("expect constant values")) } - value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, tps[j]) + value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[indexInfo.Columns[j].Offset].FieldType) if err != nil { return nil, err } diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index 046d57b4469cc..b9c8a6648b0dc 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -16,6 +16,7 @@ package tikv import ( "bytes" "context" + "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" From b44db0b3935694735045293ac7638c8160875475 Mon Sep 17 00:00:00 2001 From: crazycs Date: Sun, 28 Apr 2019 10:26:42 +0800 Subject: [PATCH 18/22] use session variable tidb_wait_table_split_finish to chose sync or async split region. --- executor/split.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/executor/split.go b/executor/split.go index 93a7fe5d830af..bdfb8ad641a0e 100644 --- a/executor/split.go +++ b/executor/split.go @@ -66,6 +66,9 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) e regionIDs = append(regionIDs, regionID) } + if !e.ctx.GetSessionVars().WaitTableSplitFinish { + return nil + } for _, regionID := range regionIDs { err := s.WaitScatterRegionFinish(regionID) if err != nil { From 15c951462d6570a57d0b12d569406841291056c6 Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 29 Apr 2019 09:15:53 +0800 Subject: [PATCH 19/22] update go mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 39cb92f0f4646..3e6e783d6be62 100644 --- a/go.mod +++ b/go.mod @@ -73,4 +73,4 @@ require ( sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) -replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190424150639-ab99f9af0665 +replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190429011413-f0b2f675e283 diff --git a/go.sum b/go.sum index 9d3916c828ab8..e5957841c73d6 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/crazycs520/parser v0.0.0-20190424150639-ab99f9af0665 h1:eFA/z4xCptmSRH9b3N2NnKtdr5ShMWLSoXlClAv+MCA= -github.com/crazycs520/parser v0.0.0-20190424150639-ab99f9af0665/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= +github.com/crazycs520/parser v0.0.0-20190429011413-f0b2f675e283 h1:m1tZP0PGLZ6NIuApN7HG5gqv0t+ie+HWG0wNMUXJ4m0= +github.com/crazycs520/parser v0.0.0-20190429011413-f0b2f675e283/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= From 650c42805894e3062e7d4c886c6c40734602dbf4 Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 6 May 2019 10:27:09 +0800 Subject: [PATCH 20/22] address comment --- executor/executor_test.go | 1 - executor/split.go | 3 ++- planner/core/planbuilder.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 0856d15158683..08b898966a30e 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3665,7 +3665,6 @@ func (s *testSuite) TestSplitIndexRegion(c *C) { c.Assert(err, NotNil) terr := errors.Cause(err).(*terror.Error) c.Assert(terr.Code(), Equals, terror.ErrCode(mysql.WarnDataTruncated)) - } type testOOMSuite struct { diff --git a/executor/split.go b/executor/split.go index bdfb8ad641a0e..eb4d57bde69b3 100644 --- a/executor/split.go +++ b/executor/split.go @@ -15,6 +15,7 @@ package executor import ( "context" + "math" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" @@ -50,7 +51,7 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) e regionIDs := make([]uint64, 0, len(e.valueLists)) index := tables.NewIndex(e.table.Meta().ID, e.table.Meta(), e.indexInfo) for _, values := range e.valueLists { - idxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, values, 1, nil) + idxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, values, math.MinInt64, nil) if err != nil { return err } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index f485ad5984638..acb55c4358aa9 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1609,7 +1609,7 @@ func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Pla for j, valueItem := range valuesItem { x, ok := valueItem.(*driver.ValueExpr) if !ok { - return nil, errors.Trace(errors.New("expect constant values")) + return nil, errors.New("expect constant values") } value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[indexInfo.Columns[j].Offset].FieldType) if err != nil { From 1d2e9c9dc9398a4d3dfcb6689a5e4a44a7c761cc Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 6 May 2019 14:38:44 +0800 Subject: [PATCH 21/22] address comment --- planner/core/planbuilder.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index f105dd8d790b2..118b41c8bd9d3 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1613,9 +1613,6 @@ func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Pla if indexInfo == nil { return nil, ErrKeyDoesNotExist.GenWithStackByArgs(node.IndexName, tblInfo.Name) } - schema := expression.TableInfo2SchemaWithDBName(b.ctx, node.Table.Schema, tblInfo) - mockTablePlan := LogicalTableDual{}.Init(b.ctx) - mockTablePlan.SetSchema(schema) indexValues := make([][]types.Datum, 0, len(node.ValueLists)) for i, valuesItem := range node.ValueLists { @@ -1628,7 +1625,8 @@ func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Pla if !ok { return nil, errors.New("expect constant values") } - value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[indexInfo.Columns[j].Offset].FieldType) + colOffset := indexInfo.Columns[j].Offset + value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType) if err != nil { return nil, err } From 44016a5cb8c0492e21be00531c1ec2b5b8066e1c Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 6 May 2019 17:27:53 +0800 Subject: [PATCH 22/22] address comment and update go.mod --- executor/split.go | 2 ++ go.mod | 4 +--- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/executor/split.go b/executor/split.go index eb4d57bde69b3..dc1b75e8018f3 100644 --- a/executor/split.go +++ b/executor/split.go @@ -75,6 +75,8 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) e if err != nil { logutil.Logger(context.Background()).Warn("wait scatter region failed", zap.Uint64("regionID", regionID), + zap.String("table", e.table.Meta().Name.L), + zap.String("index", e.indexInfo.Name.L), zap.Error(err)) } } diff --git a/go.mod b/go.mod index 7fce724a9cb80..70a6f40911b88 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190429120706-c378059f7f42 + github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825 github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 @@ -73,5 +73,3 @@ require ( sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) - -replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190506044601-7645e4600bc6 diff --git a/go.sum b/go.sum index 1b1b8bee49b1e..288f6e6d080d2 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,6 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/crazycs520/parser v0.0.0-20190506044601-7645e4600bc6 h1:bl4lUPgi1lXT7ZZMM0o3OnpI7NowQI+2grhbodr4qu8= -github.com/crazycs520/parser v0.0.0-20190506044601-7645e4600bc6/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= @@ -162,6 +160,8 @@ github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= +github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825 h1:U9Kdnknj4n2v76Mg7wazevZ5N9U1OIaMwSNRVLEcLX0= +github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 h1:ZoKjndm/Ig7Ru/wojrQkc/YLUttUdQXoH77gtuWCvL4= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669/go.mod h1:MUCxRzOkYiWZtlyi4MhxjCIj9PgQQ/j+BLNGm7aUsnM= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU=