-
Notifications
You must be signed in to change notification settings - Fork 5.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
store/copr: balance region for batch cop task #24521
Changes from 15 commits
87e8db4
6ad74a5
d33d2a0
60c3621
fc86ee7
0e4e72c
9d7bb1e
0cf711a
0bf0874
39c8786
9ffb7a6
ccbb3f3
62f2a13
9f32a6b
3a641fc
416378f
b92fb5d
2986c4e
c86d192
8af078f
1859b5b
a7eb17f
1023411
09d01f6
d682af3
624abed
56140a3
e1cdae2
2fbe914
ddd0700
8493913
5be4895
f1de8b4
3a76a3e
ee94b2d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -11,53 +11,59 @@ | |
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package copr | ||
package tikv | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why do we change the package belonging? batch cop only involves cop requests, so putting it copr package is reasonable. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For 2 reasons I move the package:
|
||
|
||
import ( | ||
"context" | ||
"sync/atomic" | ||
"time" | ||
|
||
"github.com/pingcap/errors" | ||
"github.com/pingcap/tidb/store/tikv" | ||
"github.com/pingcap/kvproto/pkg/metapb" | ||
tikverr "github.com/pingcap/tidb/store/tikv/error" | ||
"github.com/pingcap/tidb/store/tikv/tikvrpc" | ||
"google.golang.org/grpc/codes" | ||
"google.golang.org/grpc/status" | ||
) | ||
|
||
type RegionInfo struct { | ||
Region RegionVerID | ||
Meta *metapb.Region | ||
Ranges *KeyRanges | ||
AllStores []uint64 | ||
} | ||
|
||
// RegionBatchRequestSender sends BatchCop requests to TiFlash server by stream way. | ||
type RegionBatchRequestSender struct { | ||
*tikv.RegionRequestSender | ||
*RegionRequestSender | ||
} | ||
|
||
// NewRegionBatchRequestSender creates a RegionBatchRequestSender object. | ||
func NewRegionBatchRequestSender(cache *tikv.RegionCache, client tikv.Client) *RegionBatchRequestSender { | ||
func NewRegionBatchRequestSender(cache *RegionCache, client Client) *RegionBatchRequestSender { | ||
return &RegionBatchRequestSender{ | ||
RegionRequestSender: tikv.NewRegionRequestSender(cache, client), | ||
RegionRequestSender: NewRegionRequestSender(cache, client), | ||
} | ||
} | ||
|
||
func (ss *RegionBatchRequestSender) sendStreamReqToAddr(bo *backoffer, ctxs []copTaskAndRPCContext, req *tikvrpc.Request, timout time.Duration) (resp *tikvrpc.Response, retry bool, cancel func(), err error) { | ||
func (ss *RegionBatchRequestSender) SendReqToAddr(bo *Backoffer, rpcCtx *RPCContext, regionInfos []RegionInfo, req *tikvrpc.Request, timout time.Duration) (resp *tikvrpc.Response, retry bool, cancel func(), err error) { | ||
// use the first ctx to send request, because every ctx has same address. | ||
cancel = func() {} | ||
rpcCtx := ctxs[0].ctx | ||
if e := tikvrpc.SetContext(req, rpcCtx.Meta, rpcCtx.Peer); e != nil { | ||
return nil, false, cancel, errors.Trace(e) | ||
} | ||
ctx := bo.GetCtx() | ||
if rawHook := ctx.Value(tikv.RPCCancellerCtxKey{}); rawHook != nil { | ||
ctx, cancel = rawHook.(*tikv.RPCCanceller).WithCancel(ctx) | ||
if rawHook := ctx.Value(RPCCancellerCtxKey{}); rawHook != nil { | ||
ctx, cancel = rawHook.(*RPCCanceller).WithCancel(ctx) | ||
} | ||
start := time.Now() | ||
resp, err = ss.GetClient().SendRequest(ctx, rpcCtx.Addr, req, timout) | ||
if ss.Stats != nil { | ||
tikv.RecordRegionRequestRuntimeStats(ss.Stats, req.Type, time.Since(start)) | ||
RecordRegionRequestRuntimeStats(ss.Stats, req.Type, time.Since(start)) | ||
} | ||
if err != nil { | ||
cancel() | ||
ss.SetRPCError(err) | ||
e := ss.onSendFail(bo, ctxs, err) | ||
e := ss.onSendFailForBatchRegions(bo, rpcCtx, regionInfos, err) | ||
if e != nil { | ||
return nil, false, func() {}, errors.Trace(e) | ||
} | ||
|
@@ -67,30 +73,25 @@ func (ss *RegionBatchRequestSender) sendStreamReqToAddr(bo *backoffer, ctxs []co | |
return | ||
} | ||
|
||
func (ss *RegionBatchRequestSender) onSendFail(bo *backoffer, ctxs []copTaskAndRPCContext, err error) error { | ||
func (ss *RegionBatchRequestSender) onSendFailForBatchRegions(bo *Backoffer, ctx *RPCContext, regionInfos []RegionInfo, err error) error { | ||
// If it failed because the context is cancelled by ourself, don't retry. | ||
if errors.Cause(err) == context.Canceled || status.Code(errors.Cause(err)) == codes.Canceled { | ||
return errors.Trace(err) | ||
} else if atomic.LoadUint32(&tikv.ShuttingDown) > 0 { | ||
} else if atomic.LoadUint32(&ShuttingDown) > 0 { | ||
return tikverr.ErrTiDBShuttingDown | ||
} | ||
|
||
for _, failedCtx := range ctxs { | ||
ctx := failedCtx.ctx | ||
if ctx.Meta != nil { | ||
// The reload region param is always true. Because that every time we try, we must | ||
// re-build the range then re-create the batch sender. As a result, the len of "failStores" | ||
// will change. If tiflash's replica is more than two, the "reload region" will always be false. | ||
// Now that the batch cop and mpp has a relative low qps, it's reasonable to reload every time | ||
// when meeting io error. | ||
ss.GetRegionCache().OnSendFail(bo.TiKVBackoffer(), ctx, true, err) | ||
} | ||
} | ||
// The reload region param is always true. Because that every time we try, we must | ||
// re-build the range then re-create the batch sender. As a result, the len of "failStores" | ||
// will change. If tiflash's replica is more than two, the "reload region" will always be false. | ||
// Now that the batch cop and mpp has a relative low qps, it's reasonable to reload every time | ||
// when meeting io error. | ||
ss.GetRegionCache().OnSendFailForBatchRegions(bo, ctx.Store, regionInfos, true, err) | ||
|
||
// Retry on send request failure when it's not canceled. | ||
// When a store is not available, the leader of related region should be elected quickly. | ||
// TODO: the number of retry time should be limited:since region may be unavailable | ||
// when some unrecoverable disaster happened. | ||
err = bo.Backoff(tikv.BoTiFlashRPC, errors.Errorf("send tikv request error: %v, ctxs: %v, try next peer later", err, ctxs)) | ||
err = bo.Backoff(BoTiFlashRPC, errors.Errorf("send request error: %v, ctx: %v, regionInfos: %v", err, ctx, regionInfos)) | ||
return errors.Trace(err) | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
refactor these code to make it clearer. It is a typical problem, may be a better algo to solve it.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In my opinion, the optimistic algo is equal to a match problem or max flow problem, of which time complexity is expensive(O(V^2) for match prob and O(V^3) for max-flow prob). So the greedy method is ok with me.