From fee2a12d69fcfecad5bf0edcb484a360bdcf5bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=B1=E5=B2=9A?= <36239017+YuJuncen@users.noreply.github.com> Date: Wed, 13 Jul 2022 11:07:05 +0800 Subject: [PATCH 01/27] log-backup: implement the checkpoint V3 (#36114) close pingcap/tidb#35164 --- br/cmd/br/stream.go | 24 + br/pkg/conn/conn.go | 214 +------- br/pkg/logutil/logging.go | 27 + br/pkg/restore/client.go | 31 +- br/pkg/restore/import_retry.go | 1 + br/pkg/stream/rewrite_meta_rawkv.go | 28 - br/pkg/stream/stream_misc_test.go | 3 +- br/pkg/stream/stream_status.go | 37 +- br/pkg/streamhelper/advancer.go | 514 ++++++++++++++++++ br/pkg/streamhelper/advancer_daemon.go | 81 +++ br/pkg/streamhelper/advancer_env.go | 107 ++++ br/pkg/streamhelper/advancer_test.go | 185 +++++++ br/pkg/streamhelper/basic_lib_for_test.go | 432 +++++++++++++++ br/pkg/{stream => streamhelper}/client.go | 60 +- br/pkg/streamhelper/collector.go | 315 +++++++++++ br/pkg/streamhelper/config/advancer_conf.go | 82 +++ .../integration_test.go | 84 ++- br/pkg/{stream => streamhelper}/models.go | 18 +- .../prefix_scanner.go | 2 +- br/pkg/streamhelper/regioniter.go | 122 +++++ br/pkg/streamhelper/stream_listener.go | 170 ++++++ br/pkg/streamhelper/tsheap.go | 216 ++++++++ br/pkg/streamhelper/tsheap_test.go | 161 ++++++ br/pkg/task/stream.go | 44 +- br/pkg/utils/store_manager.go | 244 +++++++++ br/pkg/utils/worker.go | 23 + config/config.go | 16 +- domain/domain.go | 25 + go.mod | 2 +- go.sum | 4 +- metrics/log_backup.go | 51 ++ metrics/metrics.go | 4 + 32 files changed, 3043 insertions(+), 284 deletions(-) create mode 100644 br/pkg/streamhelper/advancer.go create mode 100644 br/pkg/streamhelper/advancer_daemon.go create mode 100644 br/pkg/streamhelper/advancer_env.go create mode 100644 br/pkg/streamhelper/advancer_test.go create mode 100644 br/pkg/streamhelper/basic_lib_for_test.go rename br/pkg/{stream => streamhelper}/client.go (90%) create mode 100644 br/pkg/streamhelper/collector.go create mode 100644 br/pkg/streamhelper/config/advancer_conf.go rename br/pkg/{stream => streamhelper}/integration_test.go (68%) rename br/pkg/{stream => streamhelper}/models.go (92%) rename br/pkg/{stream => streamhelper}/prefix_scanner.go (99%) create mode 100644 br/pkg/streamhelper/regioniter.go create mode 100644 br/pkg/streamhelper/stream_listener.go create mode 100644 br/pkg/streamhelper/tsheap.go create mode 100644 br/pkg/streamhelper/tsheap_test.go create mode 100644 br/pkg/utils/store_manager.go create mode 100644 metrics/log_backup.go diff --git a/br/cmd/br/stream.go b/br/cmd/br/stream.go index c59ae6d859af0..f452e38917ea5 100644 --- a/br/cmd/br/stream.go +++ b/br/cmd/br/stream.go @@ -16,6 +16,7 @@ package main import ( "github.com/pingcap/errors" + advancercfg "github.com/pingcap/tidb/br/pkg/streamhelper/config" "github.com/pingcap/tidb/br/pkg/task" "github.com/pingcap/tidb/br/pkg/trace" "github.com/pingcap/tidb/br/pkg/utils" @@ -49,6 +50,7 @@ func NewStreamCommand() *cobra.Command { newStreamStatusCommand(), newStreamTruncateCommand(), newStreamCheckCommand(), + newStreamAdvancerCommand(), ) command.SetHelpFunc(func(command *cobra.Command, strings []string) { task.HiddenFlagsForStream(command.Root().PersistentFlags()) @@ -157,6 +159,21 @@ func newStreamCheckCommand() *cobra.Command { return command } +func newStreamAdvancerCommand() *cobra.Command { + command := &cobra.Command{ + Use: "advancer", + Short: "Start a central worker for advancing the checkpoint. (only for debuging, this subcommand should be integrated to TiDB)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return streamCommand(cmd, task.StreamCtl) + }, + Hidden: true, + } + task.DefineStreamCommonFlags(command.Flags()) + advancercfg.DefineFlagsForCheckpointAdvancerConfig(command.Flags()) + return command +} + func streamCommand(command *cobra.Command, cmdName string) error { var cfg task.StreamConfig var err error @@ -192,6 +209,13 @@ func streamCommand(command *cobra.Command, cmdName string) error { if err = cfg.ParseStreamPauseFromFlags(command.Flags()); err != nil { return errors.Trace(err) } + case task.StreamCtl: + if err = cfg.ParseStreamCommonFromFlags(command.Flags()); err != nil { + return errors.Trace(err) + } + if err = cfg.AdvancerCfg.GetFromFlags(command.Flags()); err != nil { + return errors.Trace(err) + } default: if err = cfg.ParseStreamCommonFromFlags(command.Flags()); err != nil { return errors.Trace(err) diff --git a/br/pkg/conn/conn.go b/br/pkg/conn/conn.go index 75eef2c1555ab..f90743e1bd3d5 100755 --- a/br/pkg/conn/conn.go +++ b/br/pkg/conn/conn.go @@ -9,16 +9,14 @@ import ( "fmt" "net/http" "net/url" - "os" "strings" - "sync" - "time" "github.com/docker/go-units" "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" backuppb "github.com/pingcap/kvproto/pkg/brpb" + logbackup "github.com/pingcap/kvproto/pkg/logbackuppb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" berrors "github.com/pingcap/tidb/br/pkg/errors" @@ -35,9 +33,7 @@ import ( pd "github.com/tikv/pd/client" "go.uber.org/zap" "google.golang.org/grpc" - "google.golang.org/grpc/backoff" "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/status" ) @@ -49,83 +45,17 @@ const ( // DefaultMergeRegionKeyCount is the default region key count, 960000. DefaultMergeRegionKeyCount uint64 = 960000 - - dialTimeout = 30 * time.Second - - resetRetryTimes = 3 ) -// Pool is a lazy pool of gRPC channels. -// When `Get` called, it lazily allocates new connection if connection not full. -// If it's full, then it will return allocated channels round-robin. -type Pool struct { - mu sync.Mutex - - conns []*grpc.ClientConn - next int - cap int - newConn func(ctx context.Context) (*grpc.ClientConn, error) -} - -func (p *Pool) takeConns() (conns []*grpc.ClientConn) { - p.mu.Lock() - defer p.mu.Unlock() - p.conns, conns = nil, p.conns - p.next = 0 - return conns -} - -// Close closes the conn pool. -func (p *Pool) Close() { - for _, c := range p.takeConns() { - if err := c.Close(); err != nil { - log.Warn("failed to close clientConn", zap.String("target", c.Target()), zap.Error(err)) - } - } -} - -// Get tries to get an existing connection from the pool, or make a new one if the pool not full. -func (p *Pool) Get(ctx context.Context) (*grpc.ClientConn, error) { - p.mu.Lock() - defer p.mu.Unlock() - if len(p.conns) < p.cap { - c, err := p.newConn(ctx) - if err != nil { - return nil, err - } - p.conns = append(p.conns, c) - return c, nil - } - - conn := p.conns[p.next] - p.next = (p.next + 1) % p.cap - return conn, nil -} - -// NewConnPool creates a new Pool by the specified conn factory function and capacity. -func NewConnPool(capacity int, newConn func(ctx context.Context) (*grpc.ClientConn, error)) *Pool { - return &Pool{ - cap: capacity, - conns: make([]*grpc.ClientConn, 0, capacity), - newConn: newConn, - - mu: sync.Mutex{}, - } -} - // Mgr manages connections to a TiDB cluster. type Mgr struct { *pdutil.PdController - tlsConf *tls.Config - dom *domain.Domain - storage kv.Storage // Used to access SQL related interfaces. - tikvStore tikv.Storage // Used to access TiKV specific interfaces. - grpcClis struct { - mu sync.Mutex - clis map[uint64]*grpc.ClientConn - } - keepalive keepalive.ClientParameters + dom *domain.Domain + storage kv.Storage // Used to access SQL related interfaces. + tikvStore tikv.Storage // Used to access TiKV specific interfaces. ownsStorage bool + + *utils.StoreManager } // StoreBehavior is the action to do in GetAllTiKVStores when a non-TiKV @@ -298,122 +228,31 @@ func NewMgr( storage: storage, tikvStore: tikvStorage, dom: dom, - tlsConf: tlsConf, ownsStorage: g.OwnsStorage(), - grpcClis: struct { - mu sync.Mutex - clis map[uint64]*grpc.ClientConn - }{clis: make(map[uint64]*grpc.ClientConn)}, - keepalive: keepalive, + StoreManager: utils.NewStoreManager(controller.GetPDClient(), keepalive, tlsConf), } return mgr, nil } -func (mgr *Mgr) getGrpcConnLocked(ctx context.Context, storeID uint64) (*grpc.ClientConn, error) { - failpoint.Inject("hint-get-backup-client", func(v failpoint.Value) { - log.Info("failpoint hint-get-backup-client injected, "+ - "process will notify the shell.", zap.Uint64("store", storeID)) - if sigFile, ok := v.(string); ok { - file, err := os.Create(sigFile) - if err != nil { - log.Warn("failed to create file for notifying, skipping notify", zap.Error(err)) - } - if file != nil { - file.Close() - } - } - time.Sleep(3 * time.Second) - }) - store, err := mgr.GetPDClient().GetStore(ctx, storeID) - if err != nil { - return nil, errors.Trace(err) - } - opt := grpc.WithInsecure() - if mgr.tlsConf != nil { - opt = grpc.WithTransportCredentials(credentials.NewTLS(mgr.tlsConf)) - } - ctx, cancel := context.WithTimeout(ctx, dialTimeout) - bfConf := backoff.DefaultConfig - bfConf.MaxDelay = time.Second * 3 - addr := store.GetPeerAddress() - if addr == "" { - addr = store.GetAddress() - } - conn, err := grpc.DialContext( - ctx, - addr, - opt, - grpc.WithBlock(), - grpc.WithConnectParams(grpc.ConnectParams{Backoff: bfConf}), - grpc.WithKeepaliveParams(mgr.keepalive), - ) - cancel() - if err != nil { - return nil, berrors.ErrFailedToConnect.Wrap(err).GenWithStack("failed to make connection to store %d", storeID) - } - return conn, nil -} - // GetBackupClient get or create a backup client. func (mgr *Mgr) GetBackupClient(ctx context.Context, storeID uint64) (backuppb.BackupClient, error) { - if ctx.Err() != nil { - return nil, errors.Trace(ctx.Err()) - } - - mgr.grpcClis.mu.Lock() - defer mgr.grpcClis.mu.Unlock() - - if conn, ok := mgr.grpcClis.clis[storeID]; ok { - // Find a cached backup client. - return backuppb.NewBackupClient(conn), nil - } - - conn, err := mgr.getGrpcConnLocked(ctx, storeID) - if err != nil { - return nil, errors.Trace(err) + var cli backuppb.BackupClient + if err := mgr.WithConn(ctx, storeID, func(cc *grpc.ClientConn) { + cli = backuppb.NewBackupClient(cc) + }); err != nil { + return nil, err } - // Cache the conn. - mgr.grpcClis.clis[storeID] = conn - return backuppb.NewBackupClient(conn), nil + return cli, nil } -// ResetBackupClient reset the connection for backup client. -func (mgr *Mgr) ResetBackupClient(ctx context.Context, storeID uint64) (backuppb.BackupClient, error) { - if ctx.Err() != nil { - return nil, errors.Trace(ctx.Err()) - } - - mgr.grpcClis.mu.Lock() - defer mgr.grpcClis.mu.Unlock() - - if conn, ok := mgr.grpcClis.clis[storeID]; ok { - // Find a cached backup client. - log.Info("Reset backup client", zap.Uint64("storeID", storeID)) - err := conn.Close() - if err != nil { - log.Warn("close backup connection failed, ignore it", zap.Uint64("storeID", storeID)) - } - delete(mgr.grpcClis.clis, storeID) - } - var ( - conn *grpc.ClientConn - err error - ) - for retry := 0; retry < resetRetryTimes; retry++ { - conn, err = mgr.getGrpcConnLocked(ctx, storeID) - if err != nil { - log.Warn("failed to reset grpc connection, retry it", - zap.Int("retry time", retry), logutil.ShortError(err)) - time.Sleep(time.Duration(retry+3) * time.Second) - continue - } - mgr.grpcClis.clis[storeID] = conn - break - } - if err != nil { - return nil, errors.Trace(err) +func (mgr *Mgr) GetLogBackupClient(ctx context.Context, storeID uint64) (logbackup.LogBackupClient, error) { + var cli logbackup.LogBackupClient + if err := mgr.WithConn(ctx, storeID, func(cc *grpc.ClientConn) { + cli = logbackup.NewLogBackupClient(cc) + }); err != nil { + return nil, err } - return backuppb.NewBackupClient(conn), nil + return cli, nil } // GetStorage returns a kv storage. @@ -423,7 +262,7 @@ func (mgr *Mgr) GetStorage() kv.Storage { // GetTLSConfig returns the tls config. func (mgr *Mgr) GetTLSConfig() *tls.Config { - return mgr.tlsConf + return mgr.StoreManager.TLSConfig() } // GetLockResolver gets the LockResolver. @@ -436,17 +275,10 @@ func (mgr *Mgr) GetDomain() *domain.Domain { return mgr.dom } -// Close closes all client in Mgr. func (mgr *Mgr) Close() { - mgr.grpcClis.mu.Lock() - for _, cli := range mgr.grpcClis.clis { - err := cli.Close() - if err != nil { - log.Error("fail to close Mgr", zap.Error(err)) - } + if mgr.StoreManager != nil { + mgr.StoreManager.Close() } - mgr.grpcClis.mu.Unlock() - // Gracefully shutdown domain so it does not affect other TiDB DDL. // Must close domain before closing storage, otherwise it gets stuck forever. if mgr.ownsStorage { diff --git a/br/pkg/logutil/logging.go b/br/pkg/logutil/logging.go index 71b882b7af9db..354b900e5605a 100644 --- a/br/pkg/logutil/logging.go +++ b/br/pkg/logutil/logging.go @@ -14,6 +14,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/pingcap/tidb/br/pkg/redact" + "github.com/pingcap/tidb/kv" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -269,3 +270,29 @@ func Redact(field zap.Field) zap.Field { } return field } + +// StringifyRanges wrappes the key range into a stringer. +type StringifyKeys []kv.KeyRange + +func (kr StringifyKeys) String() string { + sb := new(strings.Builder) + sb.WriteString("{") + for i, rng := range kr { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString("[") + sb.WriteString(redact.Key(rng.StartKey)) + sb.WriteString(", ") + var endKey string + if len(rng.EndKey) == 0 { + endKey = "inf" + } else { + endKey = redact.Key(rng.EndKey) + } + sb.WriteString(redact.String(endKey)) + sb.WriteString(")") + } + sb.WriteString("}") + return sb.String() +} diff --git a/br/pkg/restore/client.go b/br/pkg/restore/client.go index 4a0896e1c7900..e5f382233f94a 100644 --- a/br/pkg/restore/client.go +++ b/br/pkg/restore/client.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/kvproto/pkg/import_sstpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" + "github.com/pingcap/tidb/br/pkg/backup" "github.com/pingcap/tidb/br/pkg/checksum" "github.com/pingcap/tidb/br/pkg/conn" berrors "github.com/pingcap/tidb/br/pkg/errors" @@ -2174,7 +2175,7 @@ func (rc *Client) SaveSchemas( m.StartVersion = logStartTS }) - schemas := sr.TidyOldSchemas() + schemas := TidyOldSchemas(sr) schemasConcurrency := uint(mathutil.Min(64, schemas.Len())) err := schemas.BackupSchemas(ctx, metaWriter, nil, nil, rc.restoreTS, schemasConcurrency, 0, true, nil) if err != nil { @@ -2191,3 +2192,31 @@ func (rc *Client) SaveSchemas( func MockClient(dbs map[string]*utils.Database) *Client { return &Client{databases: dbs} } + +// TidyOldSchemas produces schemas information. +func TidyOldSchemas(sr *stream.SchemasReplace) *backup.Schemas { + var schemaIsEmpty bool + schemas := backup.NewBackupSchemas() + + for _, dr := range sr.DbMap { + if dr.OldDBInfo == nil { + continue + } + + schemaIsEmpty = true + for _, tr := range dr.TableMap { + if tr.OldTableInfo == nil { + continue + } + schemas.AddSchema(dr.OldDBInfo, tr.OldTableInfo) + schemaIsEmpty = false + } + + // backup this empty schema if it has nothing table. + if schemaIsEmpty { + schemas.AddSchema(dr.OldDBInfo, nil) + } + } + return schemas + +} diff --git a/br/pkg/restore/import_retry.go b/br/pkg/restore/import_retry.go index 20d613d9bbd31..17c706c9e4444 100644 --- a/br/pkg/restore/import_retry.go +++ b/br/pkg/restore/import_retry.go @@ -234,6 +234,7 @@ func (r *RPCResult) StrategyForRetryGoError() RetryStrategy { if r.Err == nil { return StrategyGiveUp } + // we should unwrap the error or we cannot get the write gRPC status. if gRPCErr, ok := status.FromError(errors.Cause(r.Err)); ok { switch gRPCErr.Code() { diff --git a/br/pkg/stream/rewrite_meta_rawkv.go b/br/pkg/stream/rewrite_meta_rawkv.go index 56b866314b3ea..84f1b3d200048 100644 --- a/br/pkg/stream/rewrite_meta_rawkv.go +++ b/br/pkg/stream/rewrite_meta_rawkv.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" - "github.com/pingcap/tidb/br/pkg/backup" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/parser/model" @@ -98,33 +97,6 @@ func NewSchemasReplace( } } -// TidyOldSchemas produces schemas information. -func (sr *SchemasReplace) TidyOldSchemas() *backup.Schemas { - var schemaIsEmpty bool - schemas := backup.NewBackupSchemas() - - for _, dr := range sr.DbMap { - if dr.OldDBInfo == nil { - continue - } - - schemaIsEmpty = true - for _, tr := range dr.TableMap { - if tr.OldTableInfo == nil { - continue - } - schemas.AddSchema(dr.OldDBInfo, tr.OldTableInfo) - schemaIsEmpty = false - } - - // backup this empty schema if it has nothing table. - if schemaIsEmpty { - schemas.AddSchema(dr.OldDBInfo, nil) - } - } - return schemas -} - func (sr *SchemasReplace) rewriteKeyForDB(key []byte, cf string) ([]byte, bool, error) { rawMetaKey, err := ParseTxnMetaKeyFrom(key) if err != nil { diff --git a/br/pkg/stream/stream_misc_test.go b/br/pkg/stream/stream_misc_test.go index ac31254ffd641..3a057ed2a16df 100644 --- a/br/pkg/stream/stream_misc_test.go +++ b/br/pkg/stream/stream_misc_test.go @@ -7,6 +7,7 @@ import ( backuppb "github.com/pingcap/kvproto/pkg/brpb" "github.com/pingcap/tidb/br/pkg/stream" + "github.com/pingcap/tidb/br/pkg/streamhelper" "github.com/stretchr/testify/require" ) @@ -15,7 +16,7 @@ func TestGetCheckpointOfTask(t *testing.T) { Info: backuppb.StreamBackupTaskInfo{ StartTs: 8, }, - Checkpoints: []stream.Checkpoint{ + Checkpoints: []streamhelper.Checkpoint{ { ID: 1, TS: 10, diff --git a/br/pkg/stream/stream_status.go b/br/pkg/stream/stream_status.go index 70d9b1708f938..e08f3f6c34513 100644 --- a/br/pkg/stream/stream_status.go +++ b/br/pkg/stream/stream_status.go @@ -4,6 +4,7 @@ package stream import ( "context" + "crypto/tls" "encoding/json" "fmt" "io" @@ -17,12 +18,13 @@ import ( backuppb "github.com/pingcap/kvproto/pkg/brpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" - "github.com/pingcap/tidb/br/pkg/conn" "github.com/pingcap/tidb/br/pkg/glue" "github.com/pingcap/tidb/br/pkg/httputil" "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/storage" + . "github.com/pingcap/tidb/br/pkg/streamhelper" "github.com/tikv/client-go/v2/oracle" + pd "github.com/tikv/pd/client" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) @@ -104,6 +106,9 @@ func (t TaskStatus) GetMinStoreCheckpoint() Checkpoint { initialized = true checkpoint = cp } + if cp.Type() == CheckpointTypeGlobal { + return cp + } } return checkpoint } @@ -131,7 +136,6 @@ func (p *printByTable) AddTask(task TaskStatus) { info := fmt.Sprintf("%s; gap=%s", pTime, gapColor.Sprint(gap)) return info } - table.Add("checkpoint[global]", formatTS(task.GetMinStoreCheckpoint().TS)) p.addCheckpoints(&task, table, formatTS) for store, e := range task.LastErrors { table.Add(fmt.Sprintf("error[store=%d]", store), e.ErrorCode) @@ -142,11 +146,21 @@ func (p *printByTable) AddTask(task TaskStatus) { } func (p *printByTable) addCheckpoints(task *TaskStatus, table *glue.Table, formatTS func(uint64) string) { - for _, cp := range task.Checkpoints { - switch cp.Type() { - case CheckpointTypeStore: - table.Add(fmt.Sprintf("checkpoint[store=%d]", cp.ID), formatTS(cp.TS)) + cp := task.GetMinStoreCheckpoint() + items := make([][2]string, 0, len(task.Checkpoints)) + if cp.Type() != CheckpointTypeGlobal { + for _, cp := range task.Checkpoints { + switch cp.Type() { + case CheckpointTypeStore: + items = append(items, [2]string{fmt.Sprintf("checkpoint[store=%d]", cp.ID), formatTS(cp.TS)}) + } } + } else { + items = append(items, [2]string{"checkpoint[central-global]", formatTS(cp.TS)}) + } + + for _, item := range items { + table.Add(item[0], item[1]) } } @@ -241,10 +255,15 @@ func (p *printByJSON) PrintTasks() { var logCountSumRe = regexp.MustCompile(`tikv_stream_handle_kv_batch_sum ([0-9]+)`) +type PDInfoProvider interface { + GetPDClient() pd.Client + GetTLSConfig() *tls.Config +} + // MaybeQPS get a number like the QPS of last seconds for each store via the prometheus interface. // TODO: this is a temporary solution(aha, like in a Hackthon), // we MUST find a better way for providing this information. -func MaybeQPS(ctx context.Context, mgr *conn.Mgr) (float64, error) { +func MaybeQPS(ctx context.Context, mgr PDInfoProvider) (float64, error) { c := mgr.GetPDClient() prefix := "http://" if mgr.GetTLSConfig() != nil { @@ -316,12 +335,12 @@ func MaybeQPS(ctx context.Context, mgr *conn.Mgr) (float64, error) { // StatusController is the controller type (or context type) for the command `stream status`. type StatusController struct { meta *MetaDataClient - mgr *conn.Mgr + mgr PDInfoProvider view TaskPrinter } // NewStatusContorller make a status controller via some resource accessors. -func NewStatusController(meta *MetaDataClient, mgr *conn.Mgr, view TaskPrinter) *StatusController { +func NewStatusController(meta *MetaDataClient, mgr PDInfoProvider, view TaskPrinter) *StatusController { return &StatusController{ meta: meta, mgr: mgr, diff --git a/br/pkg/streamhelper/advancer.go b/br/pkg/streamhelper/advancer.go new file mode 100644 index 0000000000000..e20516285e962 --- /dev/null +++ b/br/pkg/streamhelper/advancer.go @@ -0,0 +1,514 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. + +package streamhelper + +import ( + "bytes" + "context" + "math" + "reflect" + "sort" + "strings" + "sync" + "time" + + "github.com/pingcap/errors" + backuppb "github.com/pingcap/kvproto/pkg/brpb" + "github.com/pingcap/log" + berrors "github.com/pingcap/tidb/br/pkg/errors" + "github.com/pingcap/tidb/br/pkg/logutil" + "github.com/pingcap/tidb/br/pkg/streamhelper/config" + "github.com/pingcap/tidb/br/pkg/utils" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/metrics" + "github.com/tikv/client-go/v2/oracle" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +// CheckpointAdvancer is the central node for advancing the checkpoint of log backup. +// It's a part of "checkpoint v3". +// Generally, it scan the regions in the task range, collect checkpoints from tikvs. +// ┌──────┐ +// ┌────►│ TiKV │ +// │ └──────┘ +// │ +// │ +// ┌──────────┐GetLastFlushTSOfRegion│ ┌──────┐ +// │ Advancer ├──────────────────────┼────►│ TiKV │ +// └────┬─────┘ │ └──────┘ +// │ │ +// │ │ +// │ │ ┌──────┐ +// │ └────►│ TiKV │ +// │ └──────┘ +// │ +// │ UploadCheckpointV3 ┌──────────────────┐ +// └─────────────────────►│ PD │ +// └──────────────────┘ +type CheckpointAdvancer struct { + env Env + + // The concurrency accessed task: + // both by the task listener and ticking. + task *backuppb.StreamBackupTaskInfo + taskMu sync.Mutex + + // the read-only config. + // once tick begin, this should not be changed for now. + cfg config.Config + + // the cache of region checkpoints. + // so we can advance only ranges with huge gap. + cache CheckpointsCache + + // the internal state of advancer. + state advancerState + // the cached last checkpoint. + // if no progress, this cache can help us don't to send useless requests. + lastCheckpoint uint64 +} + +// advancerState is the sealed type for the state of advancer. +// the advancer has two stage: full scan and update small tree. +type advancerState interface { + // Note: + // Go doesn't support sealed classes or ADTs currently. + // (it can only be used at generic constraints...) + // Leave it empty for now. + + // ~*fullScan | ~*updateSmallTree +} + +// fullScan is the initial state of advancer. +// in this stage, we would "fill" the cache: +// insert ranges that union of them become the full range of task. +type fullScan struct { + fullScanTick int +} + +// updateSmallTree is the "incremental stage" of advancer. +// we have build a "filled" cache, and we can pop a subrange of it, +// try to advance the checkpoint of those ranges. +type updateSmallTree struct { + consistencyCheckTick int +} + +// NewCheckpointAdvancer creates a checkpoint advancer with the env. +func NewCheckpointAdvancer(env Env) *CheckpointAdvancer { + return &CheckpointAdvancer{ + env: env, + cfg: config.Default(), + cache: NewCheckpoints(), + state: &fullScan{}, + } +} + +// disableCache removes the cache. +// note this won't lock the checkpoint advancer at `fullScan` state forever, +// you may need to change the config `AdvancingByCache`. +func (c *CheckpointAdvancer) disableCache() { + c.cache = NoOPCheckpointCache{} + c.state = fullScan{} +} + +// enable the cache. +// also check `AdvancingByCache` in the config. +func (c *CheckpointAdvancer) enableCache() { + c.cache = NewCheckpoints() + c.state = fullScan{} +} + +// UpdateConfig updates the config for the advancer. +// Note this should be called before starting the loop, because there isn't locks, +// TODO: support updating config when advancer starts working. +// (Maybe by applying changes at begin of ticking, and add locks.) +func (c *CheckpointAdvancer) UpdateConfig(newConf config.Config) { + needRefreshCache := newConf.AdvancingByCache != c.cfg.AdvancingByCache + c.cfg = newConf + if needRefreshCache { + if c.cfg.AdvancingByCache { + c.enableCache() + } else { + c.disableCache() + } + } +} + +// UpdateConfigWith updates the config by modifying the current config. +func (c *CheckpointAdvancer) UpdateConfigWith(f func(*config.Config)) { + cfg := c.cfg + f(&cfg) + c.UpdateConfig(cfg) +} + +// Config returns the current config. +func (c *CheckpointAdvancer) Config() config.Config { + return c.cfg +} + +// GetCheckpointInRange scans the regions in the range, +// collect them to the collector. +func (c *CheckpointAdvancer) GetCheckpointInRange(ctx context.Context, start, end []byte, collector *clusterCollector) error { + log.Debug("scanning range", logutil.Key("start", start), logutil.Key("end", end)) + iter := IterateRegion(c.env, start, end) + for !iter.Done() { + rs, err := iter.Next(ctx) + if err != nil { + return err + } + log.Debug("scan region", zap.Int("len", len(rs))) + for _, r := range rs { + err := collector.collectRegion(r) + if err != nil { + log.Warn("meet error during getting checkpoint", logutil.ShortError(err)) + return err + } + } + } + return nil +} + +func (c *CheckpointAdvancer) recordTimeCost(message string, fields ...zap.Field) func() { + now := time.Now() + label := strings.ReplaceAll(message, " ", "-") + return func() { + cost := time.Since(now) + fields = append(fields, zap.Stringer("take", cost)) + metrics.AdvancerTickDuration.WithLabelValues(label).Observe(cost.Seconds()) + log.Debug(message, fields...) + } +} + +// tryAdvance tries to advance the checkpoint ts of a set of ranges which shares the same checkpoint. +func (c *CheckpointAdvancer) tryAdvance(ctx context.Context, rst RangesSharesTS) (err error) { + defer c.recordTimeCost("try advance", zap.Uint64("checkpoint", rst.TS), zap.Int("len", len(rst.Ranges)))() + defer func() { + if err != nil { + c.cache.InsertRanges(rst) + } + }() + defer utils.PanicToErr(&err) + + ranges := CollapseRanges(len(rst.Ranges), func(i int) kv.KeyRange { return rst.Ranges[i] }) + workers := utils.NewWorkerPool(4, "sub ranges") + eg, cx := errgroup.WithContext(ctx) + collector := NewClusterCollector(ctx, c.env) + collector.setOnSuccessHook(c.cache.InsertRange) + for _, r := range ranges { + r := r + workers.ApplyOnErrorGroup(eg, func() (e error) { + defer c.recordTimeCost("get regions in range", zap.Uint64("checkpoint", rst.TS))() + defer utils.PanicToErr(&e) + return c.GetCheckpointInRange(cx, r.StartKey, r.EndKey, collector) + }) + } + err = eg.Wait() + if err != nil { + return err + } + + result, err := collector.Finish(ctx) + if err != nil { + return err + } + fr := result.FailureSubRanges + if len(fr) != 0 { + log.Debug("failure regions collected", zap.Int("size", len(fr))) + c.cache.InsertRanges(RangesSharesTS{ + TS: rst.TS, + Ranges: fr, + }) + } + return nil +} + +// CalculateGlobalCheckpointLight tries to advance the global checkpoint by the cache. +func (c *CheckpointAdvancer) CalculateGlobalCheckpointLight(ctx context.Context) (uint64, error) { + log.Info("advancer with cache: current tree", zap.Stringer("ct", c.cache)) + rsts := c.cache.PopRangesWithGapGT(config.DefaultTryAdvanceThreshold) + if len(rsts) == 0 { + return 0, nil + } + workers := utils.NewWorkerPool(uint(config.DefaultMaxConcurrencyAdvance), "regions") + eg, cx := errgroup.WithContext(ctx) + for _, rst := range rsts { + rst := rst + workers.ApplyOnErrorGroup(eg, func() (err error) { + return c.tryAdvance(cx, *rst) + }) + } + err := eg.Wait() + if err != nil { + return 0, err + } + log.Info("advancer with cache: new tree", zap.Stringer("cache", c.cache)) + ts := c.cache.CheckpointTS() + return ts, nil +} + +// CalculateGlobalCheckpoint calculates the global checkpoint, which won't use the cache. +func (c *CheckpointAdvancer) CalculateGlobalCheckpoint(ctx context.Context) (uint64, error) { + var ( + cp = uint64(math.MaxInt64) + // TODO: Use The task range here. + thisRun []kv.KeyRange = []kv.KeyRange{{}} + nextRun []kv.KeyRange + ) + defer c.recordTimeCost("record all") + cx, cancel := context.WithTimeout(ctx, c.cfg.MaxBackoffTime) + defer cancel() + for { + coll := NewClusterCollector(ctx, c.env) + coll.setOnSuccessHook(c.cache.InsertRange) + for _, u := range thisRun { + err := c.GetCheckpointInRange(cx, u.StartKey, u.EndKey, coll) + if err != nil { + return 0, err + } + } + result, err := coll.Finish(ctx) + if err != nil { + return 0, err + } + log.Debug("full: a run finished", zap.Any("checkpoint", result)) + + nextRun = append(nextRun, result.FailureSubRanges...) + if cp > result.Checkpoint { + cp = result.Checkpoint + } + if len(nextRun) == 0 { + return cp, nil + } + thisRun = nextRun + nextRun = nil + log.Debug("backoffing with subranges", zap.Int("subranges", len(thisRun))) + time.Sleep(c.cfg.BackoffTime) + } +} + +// CollapseRanges collapse ranges overlapping or adjacent. +// Example: +// CollapseRanges({[1, 4], [2, 8], [3, 9]}) == {[1, 9]} +// CollapseRanges({[1, 3], [4, 7], [2, 3]}) == {[1, 3], [4, 7]} +func CollapseRanges(length int, getRange func(int) kv.KeyRange) []kv.KeyRange { + frs := make([]kv.KeyRange, 0, length) + for i := 0; i < length; i++ { + frs = append(frs, getRange(i)) + } + + sort.Slice(frs, func(i, j int) bool { + return bytes.Compare(frs[i].StartKey, frs[j].StartKey) < 0 + }) + + result := make([]kv.KeyRange, 0, len(frs)) + i := 0 + for i < len(frs) { + item := frs[i] + for { + i++ + if i >= len(frs) || (len(item.EndKey) != 0 && bytes.Compare(frs[i].StartKey, item.EndKey) > 0) { + break + } + if len(item.EndKey) != 0 && bytes.Compare(item.EndKey, frs[i].EndKey) < 0 || len(frs[i].EndKey) == 0 { + item.EndKey = frs[i].EndKey + } + } + result = append(result, item) + } + return result +} + +func (c *CheckpointAdvancer) consumeAllTask(ctx context.Context, ch <-chan TaskEvent) error { + for { + select { + case e, ok := <-ch: + if !ok { + return nil + } + log.Info("meet task event", zap.Stringer("event", &e)) + if err := c.onTaskEvent(e); err != nil { + if errors.Cause(e.Err) != context.Canceled { + log.Error("listen task meet error, would reopen.", logutil.ShortError(err)) + return err + } + return nil + } + default: + return nil + } + } +} + +// beginListenTaskChange bootstraps the initial task set, +// and returns a channel respecting the change of tasks. +func (c *CheckpointAdvancer) beginListenTaskChange(ctx context.Context) (<-chan TaskEvent, error) { + ch := make(chan TaskEvent, 1024) + if err := c.env.Begin(ctx, ch); err != nil { + return nil, err + } + err := c.consumeAllTask(ctx, ch) + if err != nil { + return nil, err + } + return ch, nil +} + +// StartTaskListener starts the task listener for the advancer. +// When no task detected, advancer would do nothing, please call this before begin the tick loop. +func (c *CheckpointAdvancer) StartTaskListener(ctx context.Context) { + cx, cancel := context.WithCancel(ctx) + var ch <-chan TaskEvent + for { + if cx.Err() != nil { + // make linter happy. + cancel() + return + } + var err error + ch, err = c.beginListenTaskChange(cx) + if err == nil { + break + } + log.Warn("failed to begin listening, retrying...", logutil.ShortError(err)) + time.Sleep(c.cfg.BackoffTime) + } + + go func() { + defer cancel() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-ch: + if !ok { + return + } + log.Info("meet task event", zap.Stringer("event", &e)) + if err := c.onTaskEvent(e); err != nil { + if errors.Cause(e.Err) != context.Canceled { + log.Error("listen task meet error, would reopen.", logutil.ShortError(err)) + time.AfterFunc(c.cfg.BackoffTime, func() { c.StartTaskListener(ctx) }) + } + return + } + } + } + }() +} + +func (c *CheckpointAdvancer) onTaskEvent(e TaskEvent) error { + c.taskMu.Lock() + defer c.taskMu.Unlock() + switch e.Type { + case EventAdd: + c.task = e.Info + case EventDel: + c.task = nil + c.state = &fullScan{} + c.cache.Clear() + case EventErr: + return e.Err + } + return nil +} + +// advanceCheckpointBy advances the checkpoint by a checkpoint getter function. +func (c *CheckpointAdvancer) advanceCheckpointBy(ctx context.Context, getCheckpoint func(context.Context) (uint64, error)) error { + start := time.Now() + cp, err := getCheckpoint(ctx) + if err != nil { + return err + } + if cp < c.lastCheckpoint { + log.Warn("failed to update global checkpoint: stale", zap.Uint64("old", c.lastCheckpoint), zap.Uint64("new", cp)) + } + if cp <= c.lastCheckpoint { + return nil + } + + log.Info("uploading checkpoint for task", + zap.Stringer("checkpoint", oracle.GetTimeFromTS(cp)), + zap.Uint64("checkpoint", cp), + zap.String("task", c.task.Name), + zap.Stringer("take", time.Since(start))) + if err := c.env.UploadV3GlobalCheckpointForTask(ctx, c.task.Name, cp); err != nil { + return errors.Annotate(err, "failed to upload global checkpoint") + } + c.lastCheckpoint = cp + metrics.LastCheckpoint.WithLabelValues(c.task.GetName()).Set(float64(c.lastCheckpoint)) + return nil +} + +// OnTick advances the inner logic clock for the advancer. +// It's synchronous: this would only return after the events triggered by the clock has all been done. +// It's generally panic-free, you may not need to trying recover a panic here. +func (c *CheckpointAdvancer) OnTick(ctx context.Context) (err error) { + defer c.recordTimeCost("tick")() + defer func() { + e := recover() + if e != nil { + log.Error("panic during handing tick", zap.Stack("stack"), logutil.ShortError(err)) + err = errors.Annotatef(berrors.ErrUnknown, "panic during handling tick: %s", e) + } + }() + err = c.tick(ctx) + return +} + +func (c *CheckpointAdvancer) onConsistencyCheckTick(s *updateSmallTree) error { + if s.consistencyCheckTick > 0 { + s.consistencyCheckTick-- + return nil + } + defer c.recordTimeCost("consistency check")() + err := c.cache.ConsistencyCheck() + if err != nil { + log.Error("consistency check failed! log backup may lose data! rolling back to full scan for saving.", logutil.ShortError(err)) + c.state = &fullScan{} + return err + } else { + log.Debug("consistency check passed.") + } + s.consistencyCheckTick = config.DefaultConsistencyCheckTick + return nil +} + +func (c *CheckpointAdvancer) tick(ctx context.Context) error { + c.taskMu.Lock() + defer c.taskMu.Unlock() + + switch s := c.state.(type) { + case *fullScan: + if s.fullScanTick > 0 { + s.fullScanTick-- + break + } + if c.task == nil { + log.Debug("No tasks yet, skipping advancing.") + return nil + } + defer func() { + s.fullScanTick = c.cfg.FullScanTick + }() + err := c.advanceCheckpointBy(ctx, c.CalculateGlobalCheckpoint) + if err != nil { + return err + } + + if c.cfg.AdvancingByCache { + c.state = &updateSmallTree{} + } + case *updateSmallTree: + if err := c.onConsistencyCheckTick(s); err != nil { + return err + } + err := c.advanceCheckpointBy(ctx, c.CalculateGlobalCheckpointLight) + if err != nil { + return err + } + default: + log.Error("Unknown state type, skipping tick", zap.Stringer("type", reflect.TypeOf(c.state))) + } + return nil +} diff --git a/br/pkg/streamhelper/advancer_daemon.go b/br/pkg/streamhelper/advancer_daemon.go new file mode 100644 index 0000000000000..909bdd85df3c6 --- /dev/null +++ b/br/pkg/streamhelper/advancer_daemon.go @@ -0,0 +1,81 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. + +package streamhelper + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/pingcap/log" + "github.com/pingcap/tidb/br/pkg/logutil" + "github.com/pingcap/tidb/metrics" + "github.com/pingcap/tidb/owner" + clientv3 "go.etcd.io/etcd/client/v3" + "go.uber.org/zap" +) + +const ( + ownerPrompt = "log-backup" + ownerPath = "/tidb/br-stream/owner" +) + +// AdvancerDaemon is a "high-availability" version of advancer. +// It involved the manager for electing a owner and doing things. +// You can embed it into your code by simply call: +// +// ad := NewAdvancerDaemon(adv, mgr) +// loop, err := ad.Begin(ctx) +// if err != nil { +// return err +// } +// loop() +type AdvancerDaemon struct { + adv *CheckpointAdvancer + manager owner.Manager +} + +func NewAdvancerDaemon(adv *CheckpointAdvancer, manager owner.Manager) *AdvancerDaemon { + return &AdvancerDaemon{ + adv: adv, + manager: manager, + } +} + +func OwnerManagerForLogBackup(ctx context.Context, etcdCli *clientv3.Client) owner.Manager { + id := uuid.New() + return owner.NewOwnerManager(ctx, etcdCli, ownerPrompt, id.String(), ownerPath) +} + +// Begin starts the daemon. +// It would do some bootstrap task, and return a closure that would begin the main loop. +func (ad *AdvancerDaemon) Begin(ctx context.Context) (func(), error) { + log.Info("begin advancer daemon", zap.String("id", ad.manager.ID())) + if err := ad.manager.CampaignOwner(); err != nil { + return nil, err + } + + ad.adv.StartTaskListener(ctx) + tick := time.NewTicker(ad.adv.cfg.TickDuration) + loop := func() { + log.Info("begin advancer daemon loop", zap.String("id", ad.manager.ID())) + for { + select { + case <-ctx.Done(): + log.Info("advancer loop exits", zap.String("id", ad.manager.ID())) + return + case <-tick.C: + log.Debug("deamon tick start", zap.Bool("is-owner", ad.manager.IsOwner())) + if ad.manager.IsOwner() { + metrics.AdvancerOwner.Set(1.0) + if err := ad.adv.OnTick(ctx); err != nil { + log.Warn("failed on tick", logutil.ShortError(err)) + } + } else { + metrics.AdvancerOwner.Set(0.0) + } + } + } + } + return loop, nil +} diff --git a/br/pkg/streamhelper/advancer_env.go b/br/pkg/streamhelper/advancer_env.go new file mode 100644 index 0000000000000..21c61ff129ce2 --- /dev/null +++ b/br/pkg/streamhelper/advancer_env.go @@ -0,0 +1,107 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. + +package streamhelper + +import ( + "context" + "time" + + logbackup "github.com/pingcap/kvproto/pkg/logbackuppb" + "github.com/pingcap/tidb/br/pkg/utils" + "github.com/pingcap/tidb/config" + pd "github.com/tikv/pd/client" + clientv3 "go.etcd.io/etcd/client/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) + +// Env is the interface required by the advancer. +type Env interface { + // The region scanner provides the region information. + RegionScanner + // LogBackupService connects to the TiKV, so we can collect the region checkpoints. + LogBackupService + // StreamMeta connects to the metadata service (normally PD). + StreamMeta +} + +// PDRegionScanner is a simple wrapper over PD +// to adapt the requirement of `RegionScan`. +type PDRegionScanner struct { + pd.Client +} + +// RegionScan gets a list of regions, starts from the region that contains key. +// Limit limits the maximum number of regions returned. +func (c PDRegionScanner) RegionScan(ctx context.Context, key []byte, endKey []byte, limit int) ([]RegionWithLeader, error) { + rs, err := c.Client.ScanRegions(ctx, key, endKey, limit) + if err != nil { + return nil, err + } + rls := make([]RegionWithLeader, 0, len(rs)) + for _, r := range rs { + rls = append(rls, RegionWithLeader{ + Region: r.Meta, + Leader: r.Leader, + }) + } + return rls, nil +} + +// clusterEnv is the environment for running in the real cluster. +type clusterEnv struct { + clis *utils.StoreManager + *TaskEventClient + PDRegionScanner +} + +// GetLogBackupClient gets the log backup client. +func (t clusterEnv) GetLogBackupClient(ctx context.Context, storeID uint64) (logbackup.LogBackupClient, error) { + var cli logbackup.LogBackupClient + err := t.clis.WithConn(ctx, storeID, func(cc *grpc.ClientConn) { + cli = logbackup.NewLogBackupClient(cc) + }) + if err != nil { + return nil, err + } + return cli, nil +} + +// CliEnv creates the Env for CLI usage. +func CliEnv(cli *utils.StoreManager, etcdCli *clientv3.Client) Env { + return clusterEnv{ + clis: cli, + TaskEventClient: &TaskEventClient{MetaDataClient: *NewMetaDataClient(etcdCli)}, + PDRegionScanner: PDRegionScanner{cli.PDClient()}, + } +} + +// TiDBEnv creates the Env by TiDB config. +func TiDBEnv(pdCli pd.Client, etcdCli *clientv3.Client, conf *config.Config) (Env, error) { + tconf, err := conf.GetTiKVConfig().Security.ToTLSConfig() + if err != nil { + return nil, err + } + return clusterEnv{ + clis: utils.NewStoreManager(pdCli, keepalive.ClientParameters{ + Time: time.Duration(conf.TiKVClient.GrpcKeepAliveTime) * time.Second, + Timeout: time.Duration(conf.TiKVClient.GrpcKeepAliveTimeout) * time.Second, + }, tconf), + TaskEventClient: &TaskEventClient{MetaDataClient: *NewMetaDataClient(etcdCli)}, + PDRegionScanner: PDRegionScanner{Client: pdCli}, + }, nil +} + +type LogBackupService interface { + // GetLogBackupClient gets the log backup client. + GetLogBackupClient(ctx context.Context, storeID uint64) (logbackup.LogBackupClient, error) +} + +// StreamMeta connects to the metadata service (normally PD). +// It provides the global checkpoint information. +type StreamMeta interface { + // Begin begins listen the task event change. + Begin(ctx context.Context, ch chan<- TaskEvent) error + // UploadV3GlobalCheckpointForTask uploads the global checkpoint to the meta store. + UploadV3GlobalCheckpointForTask(ctx context.Context, taskName string, checkpoint uint64) error +} diff --git a/br/pkg/streamhelper/advancer_test.go b/br/pkg/streamhelper/advancer_test.go new file mode 100644 index 0000000000000..f32b099069726 --- /dev/null +++ b/br/pkg/streamhelper/advancer_test.go @@ -0,0 +1,185 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. + +package streamhelper_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/tidb/br/pkg/streamhelper" + "github.com/pingcap/tidb/br/pkg/streamhelper/config" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestBasic(t *testing.T) { + c := createFakeCluster(t, 4, false) + defer func() { + fmt.Println(c) + }() + c.splitAndScatter("01", "02", "022", "023", "033", "04", "043") + ctx := context.Background() + minCheckpoint := c.advanceCheckpoints() + env := &testEnv{fakeCluster: c, testCtx: t} + adv := streamhelper.NewCheckpointAdvancer(env) + coll := streamhelper.NewClusterCollector(ctx, env) + err := adv.GetCheckpointInRange(ctx, []byte{}, []byte{}, coll) + require.NoError(t, err) + r, err := coll.Finish(ctx) + require.NoError(t, err) + require.Len(t, r.FailureSubRanges, 0) + require.Equal(t, r.Checkpoint, minCheckpoint, "%d %d", r.Checkpoint, minCheckpoint) +} + +func TestTick(t *testing.T) { + c := createFakeCluster(t, 4, false) + defer func() { + fmt.Println(c) + }() + c.splitAndScatter("01", "02", "022", "023", "033", "04", "043") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + env := &testEnv{fakeCluster: c, testCtx: t} + adv := streamhelper.NewCheckpointAdvancer(env) + adv.StartTaskListener(ctx) + adv.UpdateConfigWith(func(cac *config.Config) { + cac.FullScanTick = 0 + }) + require.NoError(t, adv.OnTick(ctx)) + for i := 0; i < 5; i++ { + cp := c.advanceCheckpoints() + require.NoError(t, adv.OnTick(ctx)) + require.Equal(t, env.getCheckpoint(), cp) + } +} + +func TestWithFailure(t *testing.T) { + log.SetLevel(zapcore.DebugLevel) + c := createFakeCluster(t, 4, true) + defer func() { + fmt.Println(c) + }() + c.splitAndScatter("01", "02", "022", "023", "033", "04", "043") + c.flushAll() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + env := &testEnv{fakeCluster: c, testCtx: t} + adv := streamhelper.NewCheckpointAdvancer(env) + adv.StartTaskListener(ctx) + adv.UpdateConfigWith(func(cac *config.Config) { + cac.FullScanTick = 0 + }) + require.NoError(t, adv.OnTick(ctx)) + + cp := c.advanceCheckpoints() + for _, v := range c.stores { + v.flush() + break + } + require.NoError(t, adv.OnTick(ctx)) + require.Less(t, env.getCheckpoint(), cp, "%d %d", env.getCheckpoint(), cp) + + for _, v := range c.stores { + v.flush() + } + + require.NoError(t, adv.OnTick(ctx)) + require.Equal(t, env.getCheckpoint(), cp) +} + +func shouldFinishInTime(t *testing.T, d time.Duration, name string, f func()) { + ch := make(chan struct{}) + go func() { + f() + close(ch) + }() + select { + case <-time.After(d): + t.Fatalf("%s should finish in %s, but not", name, d) + case <-ch: + } +} + +func TestCollectorFailure(t *testing.T) { + log.SetLevel(zapcore.DebugLevel) + c := createFakeCluster(t, 4, true) + c.onGetClient = func(u uint64) error { + return status.Error(codes.DataLoss, + "Exiled requests from the client, please slow down and listen a story: "+ + "the server has been dropped, we are longing for new nodes, however the goddess(k8s) never allocates new resource. "+ + "May you take the sword named `vim`, refactoring the definition of the nature, in the yaml file hidden at somewhere of the cluster, "+ + "to save all of us and gain the response you desiring?") + } + ctx := context.Background() + splitKeys := make([]string, 0, 10000) + for i := 0; i < 10000; i++ { + splitKeys = append(splitKeys, fmt.Sprintf("%04d", i)) + } + c.splitAndScatter(splitKeys...) + + env := &testEnv{fakeCluster: c, testCtx: t} + adv := streamhelper.NewCheckpointAdvancer(env) + coll := streamhelper.NewClusterCollector(ctx, env) + + shouldFinishInTime(t, 30*time.Second, "scan with always fail", func() { + // At this time, the sending may or may not fail because the sending and batching is doing asynchronously. + _ = adv.GetCheckpointInRange(ctx, []byte{}, []byte{}, coll) + // ...but this must fail, not getting stuck. + _, err := coll.Finish(ctx) + require.Error(t, err) + }) +} + +func oneStoreFailure() func(uint64) error { + victim := uint64(0) + mu := new(sync.Mutex) + return func(u uint64) error { + mu.Lock() + defer mu.Unlock() + if victim == 0 { + victim = u + } + if victim == u { + return status.Error(codes.NotFound, + "The place once lit by the warm lamplight has been swallowed up by the debris now.") + } + return nil + } +} + +func TestOneStoreFailure(t *testing.T) { + log.SetLevel(zapcore.DebugLevel) + c := createFakeCluster(t, 4, true) + ctx := context.Background() + splitKeys := make([]string, 0, 1000) + for i := 0; i < 1000; i++ { + splitKeys = append(splitKeys, fmt.Sprintf("%04d", i)) + } + c.splitAndScatter(splitKeys...) + c.flushAll() + + env := &testEnv{fakeCluster: c, testCtx: t} + adv := streamhelper.NewCheckpointAdvancer(env) + adv.StartTaskListener(ctx) + require.NoError(t, adv.OnTick(ctx)) + c.onGetClient = oneStoreFailure() + + for i := 0; i < 100; i++ { + c.advanceCheckpoints() + c.flushAll() + require.ErrorContains(t, adv.OnTick(ctx), "the warm lamplight") + } + + c.onGetClient = nil + cp := c.advanceCheckpoints() + c.flushAll() + require.NoError(t, adv.OnTick(ctx)) + require.Equal(t, cp, env.checkpoint) +} diff --git a/br/pkg/streamhelper/basic_lib_for_test.go b/br/pkg/streamhelper/basic_lib_for_test.go new file mode 100644 index 0000000000000..14d777f1d24e7 --- /dev/null +++ b/br/pkg/streamhelper/basic_lib_for_test.go @@ -0,0 +1,432 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. + +package streamhelper_test + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "math" + "math/rand" + "sort" + "strings" + "sync" + "testing" + + backup "github.com/pingcap/kvproto/pkg/brpb" + "github.com/pingcap/kvproto/pkg/errorpb" + logbackup "github.com/pingcap/kvproto/pkg/logbackuppb" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/tidb/br/pkg/streamhelper" + "github.com/pingcap/tidb/kv" + "google.golang.org/grpc" +) + +type flushSimulator struct { + flushedEpoch uint64 + enabled bool +} + +func (c flushSimulator) makeError(requestedEpoch uint64) *errorpb.Error { + if !c.enabled { + return nil + } + if c.flushedEpoch == 0 { + e := errorpb.Error{ + Message: "not flushed", + } + return &e + } + if c.flushedEpoch != requestedEpoch { + e := errorpb.Error{ + Message: "flushed epoch not match", + } + return &e + } + return nil +} + +func (c flushSimulator) fork() flushSimulator { + return flushSimulator{ + enabled: c.enabled, + } +} + +type region struct { + rng kv.KeyRange + leader uint64 + epoch uint64 + id uint64 + checkpoint uint64 + + fsim flushSimulator +} + +type fakeStore struct { + id uint64 + regions map[uint64]*region +} + +type fakeCluster struct { + mu sync.Mutex + idAlloced uint64 + stores map[uint64]*fakeStore + regions []*region + testCtx *testing.T + + onGetClient func(uint64) error +} + +func overlaps(a, b kv.KeyRange) bool { + if len(b.EndKey) == 0 { + return len(a.EndKey) == 0 || bytes.Compare(a.EndKey, b.StartKey) > 0 + } + if len(a.EndKey) == 0 { + return len(b.EndKey) == 0 || bytes.Compare(b.EndKey, a.StartKey) > 0 + } + return bytes.Compare(a.StartKey, b.EndKey) < 0 && bytes.Compare(b.StartKey, a.EndKey) < 0 +} + +func (f *region) splitAt(newID uint64, k string) *region { + newRegion := ®ion{ + rng: kv.KeyRange{StartKey: []byte(k), EndKey: f.rng.EndKey}, + leader: f.leader, + epoch: f.epoch + 1, + id: newID, + checkpoint: f.checkpoint, + fsim: f.fsim.fork(), + } + f.rng.EndKey = []byte(k) + f.epoch += 1 + f.fsim = f.fsim.fork() + return newRegion +} + +func (f *region) flush() { + f.fsim.flushedEpoch = f.epoch +} + +func (f *fakeStore) GetLastFlushTSOfRegion(ctx context.Context, in *logbackup.GetLastFlushTSOfRegionRequest, opts ...grpc.CallOption) (*logbackup.GetLastFlushTSOfRegionResponse, error) { + resp := &logbackup.GetLastFlushTSOfRegionResponse{ + Checkpoints: []*logbackup.RegionCheckpoint{}, + } + for _, r := range in.Regions { + region, ok := f.regions[r.Id] + if !ok || region.leader != f.id { + resp.Checkpoints = append(resp.Checkpoints, &logbackup.RegionCheckpoint{ + Err: &errorpb.Error{ + Message: "not found", + }, + Region: &logbackup.RegionIdentity{ + Id: region.id, + EpochVersion: region.epoch, + }, + }) + continue + } + if err := region.fsim.makeError(r.EpochVersion); err != nil { + resp.Checkpoints = append(resp.Checkpoints, &logbackup.RegionCheckpoint{ + Err: err, + Region: &logbackup.RegionIdentity{ + Id: region.id, + EpochVersion: region.epoch, + }, + }) + continue + } + if region.epoch != r.EpochVersion { + resp.Checkpoints = append(resp.Checkpoints, &logbackup.RegionCheckpoint{ + Err: &errorpb.Error{ + Message: "epoch not match", + }, + Region: &logbackup.RegionIdentity{ + Id: region.id, + EpochVersion: region.epoch, + }, + }) + continue + } + resp.Checkpoints = append(resp.Checkpoints, &logbackup.RegionCheckpoint{ + Checkpoint: region.checkpoint, + Region: &logbackup.RegionIdentity{ + Id: region.id, + EpochVersion: region.epoch, + }, + }) + } + return resp, nil +} + +// RegionScan gets a list of regions, starts from the region that contains key. +// Limit limits the maximum number of regions returned. +func (f *fakeCluster) RegionScan(ctx context.Context, key []byte, endKey []byte, limit int) ([]streamhelper.RegionWithLeader, error) { + f.mu.Lock() + defer f.mu.Unlock() + sort.Slice(f.regions, func(i, j int) bool { + return bytes.Compare(f.regions[i].rng.StartKey, f.regions[j].rng.StartKey) < 0 + }) + + result := make([]streamhelper.RegionWithLeader, 0, limit) + for _, region := range f.regions { + if overlaps(kv.KeyRange{StartKey: key, EndKey: endKey}, region.rng) && len(result) < limit { + regionInfo := streamhelper.RegionWithLeader{ + Region: &metapb.Region{ + Id: region.id, + StartKey: region.rng.StartKey, + EndKey: region.rng.EndKey, + RegionEpoch: &metapb.RegionEpoch{ + Version: region.epoch, + }, + }, + Leader: &metapb.Peer{ + StoreId: region.leader, + }, + } + result = append(result, regionInfo) + } else if bytes.Compare(region.rng.StartKey, key) > 0 { + break + } + } + return result, nil +} + +func (f *fakeCluster) GetLogBackupClient(ctx context.Context, storeID uint64) (logbackup.LogBackupClient, error) { + if f.onGetClient != nil { + err := f.onGetClient(storeID) + if err != nil { + return nil, err + } + } + cli, ok := f.stores[storeID] + if !ok { + f.testCtx.Fatalf("the store %d doesn't exist", storeID) + } + return cli, nil +} + +func (f *fakeCluster) findRegionById(rid uint64) *region { + for _, r := range f.regions { + if r.id == rid { + return r + } + } + return nil +} + +func (f *fakeCluster) findRegionByKey(key []byte) *region { + for _, r := range f.regions { + if bytes.Compare(key, r.rng.StartKey) >= 0 && (len(r.rng.EndKey) == 0 || bytes.Compare(key, r.rng.EndKey) < 0) { + return r + } + } + panic(fmt.Sprintf("inconsistent key space; key = %X", key)) +} + +func (f *fakeCluster) transferRegionTo(rid uint64, newPeers []uint64) { + r := f.findRegionById(rid) +storeLoop: + for _, store := range f.stores { + for _, pid := range newPeers { + if pid == store.id { + store.regions[rid] = r + continue storeLoop + } + } + delete(store.regions, rid) + } +} + +func (f *fakeCluster) splitAt(key string) { + k := []byte(key) + r := f.findRegionByKey(k) + newRegion := r.splitAt(f.idAlloc(), key) + for _, store := range f.stores { + _, ok := store.regions[r.id] + if ok { + store.regions[newRegion.id] = newRegion + } + } + f.regions = append(f.regions, newRegion) +} + +func (f *fakeCluster) idAlloc() uint64 { + f.idAlloced++ + return f.idAlloced +} + +func (f *fakeCluster) chooseStores(n int) []uint64 { + s := make([]uint64, 0, len(f.stores)) + for id := range f.stores { + s = append(s, id) + } + rand.Shuffle(len(s), func(i, j int) { + s[i], s[j] = s[j], s[i] + }) + return s[:n] +} + +func (f *fakeCluster) findPeers(rid uint64) (result []uint64) { + for _, store := range f.stores { + if _, ok := store.regions[rid]; ok { + result = append(result, store.id) + } + } + return +} + +func (f *fakeCluster) shuffleLeader(rid uint64) { + r := f.findRegionById(rid) + peers := f.findPeers(rid) + rand.Shuffle(len(peers), func(i, j int) { + peers[i], peers[j] = peers[j], peers[i] + }) + + newLeader := peers[0] + r.leader = newLeader +} + +func (f *fakeCluster) splitAndScatter(keys ...string) { + f.mu.Lock() + defer f.mu.Unlock() + for _, key := range keys { + f.splitAt(key) + } + for _, r := range f.regions { + f.transferRegionTo(r.id, f.chooseStores(3)) + f.shuffleLeader(r.id) + } +} + +// a stub once in the future we want to make different stores hold different region instances. +func (f *fakeCluster) updateRegion(rid uint64, mut func(*region)) { + r := f.findRegionById(rid) + mut(r) +} + +func (f *fakeCluster) advanceCheckpoints() uint64 { + minCheckpoint := uint64(math.MaxUint64) + for _, r := range f.regions { + f.updateRegion(r.id, func(r *region) { + r.checkpoint += rand.Uint64() % 256 + if r.checkpoint < minCheckpoint { + minCheckpoint = r.checkpoint + } + r.fsim.flushedEpoch = 0 + }) + } + return minCheckpoint +} + +func createFakeCluster(t *testing.T, n int, simEnabled bool) *fakeCluster { + c := &fakeCluster{ + stores: map[uint64]*fakeStore{}, + regions: []*region{}, + testCtx: t, + } + stores := make([]*fakeStore, 0, n) + for i := 0; i < n; i++ { + s := new(fakeStore) + s.id = c.idAlloc() + s.regions = map[uint64]*region{} + stores = append(stores, s) + } + initialRegion := ®ion{ + rng: kv.KeyRange{}, + leader: stores[0].id, + epoch: 0, + id: c.idAlloc(), + checkpoint: 0, + fsim: flushSimulator{ + enabled: simEnabled, + }, + } + for i := 0; i < 3; i++ { + if i < len(stores) { + stores[i].regions[initialRegion.id] = initialRegion + } + } + for _, s := range stores { + c.stores[s.id] = s + } + c.regions = append(c.regions, initialRegion) + return c +} + +func (r *region) String() string { + return fmt.Sprintf("%d(%d):[%s,%s);%dL%d", r.id, r.epoch, hex.EncodeToString(r.rng.StartKey), hex.EncodeToString(r.rng.EndKey), r.checkpoint, r.leader) +} + +func (s *fakeStore) String() string { + buf := new(strings.Builder) + fmt.Fprintf(buf, "%d: ", s.id) + for _, r := range s.regions { + fmt.Fprintf(buf, "%s ", r) + } + return buf.String() +} + +func (f *fakeCluster) flushAll() { + for _, r := range f.regions { + r.flush() + } +} + +func (s *fakeStore) flush() { + for _, r := range s.regions { + if r.leader == s.id { + r.flush() + } + } +} + +func (f *fakeCluster) String() string { + buf := new(strings.Builder) + fmt.Fprint(buf, ">>> fake cluster <<<\nregions: ") + for _, region := range f.regions { + fmt.Fprint(buf, region, " ") + } + fmt.Fprintln(buf) + for _, store := range f.stores { + fmt.Fprintln(buf, store) + } + return buf.String() +} + +type testEnv struct { + *fakeCluster + checkpoint uint64 + testCtx *testing.T + + mu sync.Mutex +} + +func (t *testEnv) Begin(ctx context.Context, ch chan<- streamhelper.TaskEvent) error { + tsk := streamhelper.TaskEvent{ + Type: streamhelper.EventAdd, + Name: "whole", + Info: &backup.StreamBackupTaskInfo{ + Name: "whole", + }, + } + ch <- tsk + return nil +} + +func (t *testEnv) UploadV3GlobalCheckpointForTask(ctx context.Context, _ string, checkpoint uint64) error { + t.mu.Lock() + defer t.mu.Unlock() + + if checkpoint < t.checkpoint { + t.testCtx.Fatalf("checkpoint rolling back (from %d to %d)", t.checkpoint, checkpoint) + } + t.checkpoint = checkpoint + return nil +} + +func (t *testEnv) getCheckpoint() uint64 { + t.mu.Lock() + defer t.mu.Unlock() + + return t.checkpoint +} diff --git a/br/pkg/stream/client.go b/br/pkg/streamhelper/client.go similarity index 90% rename from br/pkg/stream/client.go rename to br/pkg/streamhelper/client.go index cbeaf8a4b5437..95c5cb07e2da5 100644 --- a/br/pkg/stream/client.go +++ b/br/pkg/streamhelper/client.go @@ -1,5 +1,5 @@ // Copyright 2021 PingCAP, Inc. Licensed under Apache-2.0. -package stream +package streamhelper import ( "bytes" @@ -28,6 +28,8 @@ type Checkpoint struct { ID uint64 `json:"id,omitempty"` Version uint64 `json:"epoch_version,omitempty"` TS uint64 `json:"ts"` + + IsGlobal bool `json:"-"` } type CheckpointType int @@ -36,12 +38,15 @@ const ( CheckpointTypeStore CheckpointType = iota CheckpointTypeRegion CheckpointTypeTask + CheckpointTypeGlobal CheckpointTypeInvalid ) // Type returns the type(provider) of the checkpoint. func (cp Checkpoint) Type() CheckpointType { switch { + case cp.IsGlobal: + return CheckpointTypeGlobal case cp.ID == 0 && cp.Version == 0: return CheckpointTypeTask case cp.ID != 0 && cp.Version == 0: @@ -72,7 +77,7 @@ func ParseCheckpoint(task string, key, value []byte) (Checkpoint, error) { segs := bytes.Split(key, []byte("/")) var checkpoint Checkpoint switch string(segs[0]) { - case "store": + case checkpointTypeStore: if len(segs) != 2 { return checkpoint, errors.Annotatef(berrors.ErrPiTRMalformedMetadata, "the store checkpoint seg mismatch; segs = %v", segs) @@ -82,7 +87,9 @@ func ParseCheckpoint(task string, key, value []byte) (Checkpoint, error) { return checkpoint, err } checkpoint.ID = id - case "region": + case checkpointTypeGlobal: + checkpoint.IsGlobal = true + case checkpointTypeRegion: if len(segs) != 3 { return checkpoint, errors.Annotatef(berrors.ErrPiTRMalformedMetadata, "the region checkpoint seg mismatch; segs = %v", segs) @@ -187,6 +194,17 @@ func (c *MetaDataClient) CleanLastErrorOfTask(ctx context.Context, taskName stri return nil } +func (c *MetaDataClient) UploadV3GlobalCheckpointForTask(ctx context.Context, taskName string, checkpoint uint64) error { + key := GlobalCheckpointOf(taskName) + value := string(encodeUint64(checkpoint)) + _, err := c.KV.Put(ctx, key, value) + + if err != nil { + return err + } + return nil +} + // GetTask get the basic task handle from the metadata storage. func (c *MetaDataClient) GetTask(ctx context.Context, taskName string) (*Task, error) { resp, err := c.Get(ctx, TaskOf(taskName)) @@ -235,25 +253,35 @@ func (c *MetaDataClient) GetTaskWithPauseStatus(ctx context.Context, taskName st return &Task{cli: c, Info: taskInfo}, paused, nil } -// GetAllTasks get all of tasks from metadata storage. -func (c *MetaDataClient) GetAllTasks(ctx context.Context) ([]Task, error) { - scanner := scanEtcdPrefix(c.Client, PrefixOfTask()) - kvs, err := scanner.AllPages(ctx, 1) +func (c *MetaDataClient) TaskByInfo(t backuppb.StreamBackupTaskInfo) *Task { + return &Task{cli: c, Info: t} +} + +func (c *MetaDataClient) GetAllTasksWithRevision(ctx context.Context) ([]Task, int64, error) { + resp, err := c.KV.Get(ctx, PrefixOfTask(), clientv3.WithPrefix()) if err != nil { - return nil, errors.Trace(err) - } else if len(kvs) == 0 { - return nil, nil + return nil, 0, errors.Trace(err) + } + kvs := resp.Kvs + if len(kvs) == 0 { + return nil, resp.Header.GetRevision(), nil } tasks := make([]Task, len(kvs)) for idx, kv := range kvs { err = proto.Unmarshal(kv.Value, &tasks[idx].Info) if err != nil { - return nil, errors.Trace(err) + return nil, 0, errors.Trace(err) } tasks[idx].cli = c } - return tasks, nil + return tasks, resp.Header.GetRevision(), nil +} + +// GetAllTasks get all of tasks from metadata storage. +func (c *MetaDataClient) GetAllTasks(ctx context.Context) ([]Task, error) { + tasks, _, err := c.GetAllTasksWithRevision(ctx) + return tasks, err } // GetTaskCount get the count of tasks from metadata storage. @@ -375,6 +403,14 @@ func (t *Task) Step(ctx context.Context, store uint64, ts uint64) error { return nil } +func (t *Task) UploadGlobalCheckpoint(ctx context.Context, ts uint64) error { + _, err := t.cli.KV.Put(ctx, GlobalCheckpointOf(t.Info.Name), string(encodeUint64(ts))) + if err != nil { + return err + } + return nil +} + func (t *Task) LastError(ctx context.Context) (map[uint64]backuppb.StreamBackupError, error) { storeToError := map[uint64]backuppb.StreamBackupError{} prefix := LastErrorPrefixOf(t.Info.Name) diff --git a/br/pkg/streamhelper/collector.go b/br/pkg/streamhelper/collector.go new file mode 100644 index 0000000000000..1df39d0633d68 --- /dev/null +++ b/br/pkg/streamhelper/collector.go @@ -0,0 +1,315 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. + +package streamhelper + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync" + "sync/atomic" + + "github.com/pingcap/errors" + logbackup "github.com/pingcap/kvproto/pkg/logbackuppb" + "github.com/pingcap/log" + "github.com/pingcap/tidb/br/pkg/logutil" + "github.com/pingcap/tidb/br/pkg/utils" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/metrics" + "go.uber.org/zap" +) + +const ( + defaultBatchSize = 1024 +) + +type onSuccessHook = func(uint64, kv.KeyRange) + +// storeCollector collects the region checkpoints from some store. +// it receives requests from the input channel, batching the requests, and send them to the store. +// because the server supports batching, the range of request regions can be discrete. +// note this is a temporary struct, its lifetime is shorter that the tick of advancer. +type storeCollector struct { + storeID uint64 + batchSize int + + service LogBackupService + + input chan RegionWithLeader + // the oneshot error reporter. + err *atomic.Value + // whether the recv and send loop has exited. + doneMessenger chan struct{} + onSuccess onSuccessHook + + // concurrency safety: + // those fields should only be write on the goroutine running `recvLoop`. + // Once it exits, we can read those fields. + currentRequest logbackup.GetLastFlushTSOfRegionRequest + checkpoint uint64 + inconsistent []kv.KeyRange + regionMap map[uint64]kv.KeyRange +} + +func newStoreCollector(storeID uint64, srv LogBackupService) *storeCollector { + return &storeCollector{ + storeID: storeID, + batchSize: defaultBatchSize, + service: srv, + input: make(chan RegionWithLeader, defaultBatchSize), + err: new(atomic.Value), + doneMessenger: make(chan struct{}), + regionMap: make(map[uint64]kv.KeyRange), + } +} + +func (c *storeCollector) reportErr(err error) { + if oldErr := c.Err(); oldErr != nil { + log.Warn("reporting error twice, ignoring", logutil.AShortError("old", err), logutil.AShortError("new", oldErr)) + return + } + c.err.Store(err) +} + +func (c *storeCollector) Err() error { + err, ok := c.err.Load().(error) + if !ok { + return nil + } + return err +} + +func (c *storeCollector) setOnSuccessHook(hook onSuccessHook) { + c.onSuccess = hook +} + +func (c *storeCollector) begin(ctx context.Context) { + err := c.recvLoop(ctx) + if err != nil { + log.Warn("collector loop meet error", logutil.ShortError(err)) + c.reportErr(err) + } + close(c.doneMessenger) +} + +func (c *storeCollector) recvLoop(ctx context.Context) (err error) { + defer utils.PanicToErr(&err) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case r, ok := <-c.input: + if !ok { + return c.sendPendingRequests(ctx) + } + + if r.Leader.StoreId != c.storeID { + log.Warn("trying to request to store which isn't the leader of region.", + zap.Uint64("region", r.Region.Id), + zap.Uint64("target-store", c.storeID), + zap.Uint64("leader", r.Leader.StoreId), + ) + } + c.appendRegionMap(r) + c.currentRequest.Regions = append(c.currentRequest.Regions, &logbackup.RegionIdentity{ + Id: r.Region.GetId(), + EpochVersion: r.Region.GetRegionEpoch().GetVersion(), + }) + if len(c.currentRequest.Regions) >= c.batchSize { + err := c.sendPendingRequests(ctx) + if err != nil { + return err + } + } + } + } +} + +func (c *storeCollector) appendRegionMap(r RegionWithLeader) { + c.regionMap[r.Region.GetId()] = kv.KeyRange{StartKey: r.Region.StartKey, EndKey: r.Region.EndKey} +} + +type StoreCheckpoints struct { + HasCheckpoint bool + Checkpoint uint64 + FailureSubRanges []kv.KeyRange +} + +func (s *StoreCheckpoints) merge(other StoreCheckpoints) { + if other.HasCheckpoint && (other.Checkpoint < s.Checkpoint || !s.HasCheckpoint) { + s.Checkpoint = other.Checkpoint + s.HasCheckpoint = true + } + s.FailureSubRanges = append(s.FailureSubRanges, other.FailureSubRanges...) +} + +func (s *StoreCheckpoints) String() string { + sb := new(strings.Builder) + sb.WriteString("StoreCheckpoints:") + if s.HasCheckpoint { + sb.WriteString(strconv.Itoa(int(s.Checkpoint))) + } else { + sb.WriteString("none") + } + fmt.Fprintf(sb, ":(remaining %d ranges)", len(s.FailureSubRanges)) + return sb.String() +} + +func (c *storeCollector) spawn(ctx context.Context) func(context.Context) (StoreCheckpoints, error) { + go c.begin(ctx) + return func(cx context.Context) (StoreCheckpoints, error) { + close(c.input) + select { + case <-cx.Done(): + return StoreCheckpoints{}, cx.Err() + case <-c.doneMessenger: + } + if err := c.Err(); err != nil { + return StoreCheckpoints{}, err + } + sc := StoreCheckpoints{ + HasCheckpoint: c.checkpoint != 0, + Checkpoint: c.checkpoint, + FailureSubRanges: c.inconsistent, + } + return sc, nil + } +} + +func (c *storeCollector) sendPendingRequests(ctx context.Context) error { + log.Debug("sending batch", zap.Int("size", len(c.currentRequest.Regions)), zap.Uint64("store", c.storeID)) + cli, err := c.service.GetLogBackupClient(ctx, c.storeID) + if err != nil { + return err + } + cps, err := cli.GetLastFlushTSOfRegion(ctx, &c.currentRequest) + if err != nil { + return err + } + metrics.GetCheckpointBatchSize.WithLabelValues("checkpoint").Observe(float64(len(c.currentRequest.GetRegions()))) + c.currentRequest = logbackup.GetLastFlushTSOfRegionRequest{} + for _, checkpoint := range cps.Checkpoints { + if checkpoint.Err != nil { + log.Debug("failed to get region checkpoint", zap.Stringer("err", checkpoint.Err)) + c.inconsistent = append(c.inconsistent, c.regionMap[checkpoint.Region.Id]) + } else { + if c.onSuccess != nil { + c.onSuccess(checkpoint.Checkpoint, c.regionMap[checkpoint.Region.Id]) + } + // assuming the checkpoint would never be zero, use it as the placeholder. (1970 is so far away...) + if checkpoint.Checkpoint < c.checkpoint || c.checkpoint == 0 { + c.checkpoint = checkpoint.Checkpoint + } + } + } + return nil +} + +type runningStoreCollector struct { + collector *storeCollector + wait func(context.Context) (StoreCheckpoints, error) +} + +// clusterCollector is the controller for collecting region checkpoints for the cluster. +// It creates multi store collectors. +// ┌──────────────────────┐ Requesting ┌────────────┐ +// ┌─►│ StoreCollector[id=1] ├─────────────►│ TiKV[id=1] │ +// │ └──────────────────────┘ └────────────┘ +// │ +// │Owns +// ┌──────────────────┐ │ ┌──────────────────────┐ Requesting ┌────────────┐ +// │ ClusterCollector ├─────┼─►│ StoreCollector[id=4] ├─────────────►│ TiKV[id=4] │ +// └──────────────────┘ │ └──────────────────────┘ └────────────┘ +// │ +// │ +// │ ┌──────────────────────┐ Requesting ┌────────────┐ +// └─►│ StoreCollector[id=5] ├─────────────►│ TiKV[id=5] │ +// └──────────────────────┘ └────────────┘ +type clusterCollector struct { + mu sync.Mutex + collectors map[uint64]runningStoreCollector + noLeaders []kv.KeyRange + onSuccess onSuccessHook + + // The context for spawning sub collectors. + // Because the collectors are running lazily, + // keep the initial context for all subsequent goroutines, + // so we can make sure we can cancel all subtasks. + masterCtx context.Context + cancel context.CancelFunc + srv LogBackupService +} + +// NewClusterCollector creates a new cluster collector. +// collectors are the structure transform region information to checkpoint information, +// by requesting the checkpoint of regions in the store. +func NewClusterCollector(ctx context.Context, srv LogBackupService) *clusterCollector { + cx, cancel := context.WithCancel(ctx) + return &clusterCollector{ + collectors: map[uint64]runningStoreCollector{}, + masterCtx: cx, + cancel: cancel, + srv: srv, + } +} + +// setOnSuccessHook sets the hook when getting checkpoint of some region. +func (c *clusterCollector) setOnSuccessHook(hook onSuccessHook) { + c.onSuccess = hook +} + +// collectRegion adds a region to the collector. +func (c *clusterCollector) collectRegion(r RegionWithLeader) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.masterCtx.Err() != nil { + return nil + } + + if r.Leader.GetStoreId() == 0 { + log.Warn("there are regions without leader", zap.Uint64("region", r.Region.GetId())) + c.noLeaders = append(c.noLeaders, kv.KeyRange{StartKey: r.Region.StartKey, EndKey: r.Region.EndKey}) + return nil + } + leader := r.Leader.StoreId + _, ok := c.collectors[leader] + if !ok { + coll := newStoreCollector(leader, c.srv) + if c.onSuccess != nil { + coll.setOnSuccessHook(c.onSuccess) + } + c.collectors[leader] = runningStoreCollector{ + collector: coll, + wait: coll.spawn(c.masterCtx), + } + } + + sc := c.collectors[leader].collector + select { + case sc.input <- r: + return nil + case <-sc.doneMessenger: + err := sc.Err() + if err != nil { + c.cancel() + } + return err + } +} + +// Finish finishes collecting the region checkpoints, wait and returning the final result. +// Note this takes the ownership of this collector, you may create a new collector for next use. +func (c *clusterCollector) Finish(ctx context.Context) (StoreCheckpoints, error) { + defer c.cancel() + result := StoreCheckpoints{FailureSubRanges: c.noLeaders} + for id, coll := range c.collectors { + r, err := coll.wait(ctx) + if err != nil { + return StoreCheckpoints{}, errors.Annotatef(err, "store %d", id) + } + result.merge(r) + log.Debug("get checkpoint", zap.Stringer("checkpoint", &r), zap.Stringer("merged", &result)) + } + return result, nil +} diff --git a/br/pkg/streamhelper/config/advancer_conf.go b/br/pkg/streamhelper/config/advancer_conf.go new file mode 100644 index 0000000000000..21fac65ae0323 --- /dev/null +++ b/br/pkg/streamhelper/config/advancer_conf.go @@ -0,0 +1,82 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. + +package config + +import ( + "time" + + "github.com/spf13/pflag" +) + +const ( + flagBackoffTime = "backoff-time" + flagMaxBackoffTime = "max-backoff-time" + flagTickInterval = "tick-interval" + flagFullScanDiffTick = "full-scan-tick" + flagAdvancingByCache = "advancing-by-cache" + + DefaultConsistencyCheckTick = 5 + DefaultTryAdvanceThreshold = 3 * time.Minute +) + +var ( + DefaultMaxConcurrencyAdvance = 8 +) + +type Config struct { + // The gap between two retries. + BackoffTime time.Duration `toml:"backoff-time" json:"backoff-time"` + // When after this time we cannot collect the safe resolved ts, give up. + MaxBackoffTime time.Duration `toml:"max-backoff-time" json:"max-backoff-time"` + // The gap between calculating checkpoints. + TickDuration time.Duration `toml:"tick-interval" json:"tick-interval"` + // The backoff time of full scan. + FullScanTick int `toml:"full-scan-tick" json:"full-scan-tick"` + + // Whether enable the optimization -- use a cached heap to advancing the global checkpoint. + // This may reduce the gap of checkpoint but may cost more CPU. + AdvancingByCache bool `toml:"advancing-by-cache" json:"advancing-by-cache"` +} + +func DefineFlagsForCheckpointAdvancerConfig(f *pflag.FlagSet) { + f.Duration(flagBackoffTime, 5*time.Second, "The gap between two retries.") + f.Duration(flagMaxBackoffTime, 20*time.Minute, "After how long we should advance the checkpoint.") + f.Duration(flagTickInterval, 12*time.Second, "From how log we trigger the tick (advancing the checkpoint).") + f.Bool(flagAdvancingByCache, true, "Whether enable the optimization -- use a cached heap to advancing the global checkpoint.") + f.Int(flagFullScanDiffTick, 4, "The backoff of full scan.") +} + +func Default() Config { + return Config{ + BackoffTime: 5 * time.Second, + MaxBackoffTime: 20 * time.Minute, + TickDuration: 12 * time.Second, + FullScanTick: 4, + AdvancingByCache: true, + } +} + +func (conf *Config) GetFromFlags(f *pflag.FlagSet) error { + var err error + conf.BackoffTime, err = f.GetDuration(flagBackoffTime) + if err != nil { + return err + } + conf.MaxBackoffTime, err = f.GetDuration(flagMaxBackoffTime) + if err != nil { + return err + } + conf.TickDuration, err = f.GetDuration(flagTickInterval) + if err != nil { + return err + } + conf.FullScanTick, err = f.GetInt(flagFullScanDiffTick) + if err != nil { + return err + } + conf.AdvancingByCache, err = f.GetBool(flagAdvancingByCache) + if err != nil { + return err + } + return nil +} diff --git a/br/pkg/stream/integration_test.go b/br/pkg/streamhelper/integration_test.go similarity index 68% rename from br/pkg/stream/integration_test.go rename to br/pkg/streamhelper/integration_test.go index 92a465172afec..09f50f46e0011 100644 --- a/br/pkg/stream/integration_test.go +++ b/br/pkg/streamhelper/integration_test.go @@ -1,7 +1,7 @@ // Copyright 2021 PingCAP, Inc. Licensed under Apache-2.0. // This package tests the login in MetaClient with a embed etcd. -package stream_test +package streamhelper_test import ( "context" @@ -15,7 +15,7 @@ import ( berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/storage" - "github.com/pingcap/tidb/br/pkg/stream" + "github.com/pingcap/tidb/br/pkg/streamhelper" "github.com/pingcap/tidb/tablecodec" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/kv" @@ -63,11 +63,11 @@ func runEtcd(t *testing.T) (*embed.Etcd, *clientv3.Client) { return etcd, cli } -func simpleRanges(tableCount int) stream.Ranges { - ranges := stream.Ranges{} +func simpleRanges(tableCount int) streamhelper.Ranges { + ranges := streamhelper.Ranges{} for i := 0; i < tableCount; i++ { base := int64(i*2 + 1) - ranges = append(ranges, stream.Range{ + ranges = append(ranges, streamhelper.Range{ StartKey: tablecodec.EncodeTablePrefix(base), EndKey: tablecodec.EncodeTablePrefix(base + 1), }) @@ -75,9 +75,9 @@ func simpleRanges(tableCount int) stream.Ranges { return ranges } -func simpleTask(name string, tableCount int) stream.TaskInfo { +func simpleTask(name string, tableCount int) streamhelper.TaskInfo { backend, _ := storage.ParseBackend("noop://", nil) - task, err := stream.NewTask(name). + task, err := streamhelper.NewTask(name). FromTS(1). UntilTS(1000). WithRanges(simpleRanges(tableCount)...). @@ -110,7 +110,7 @@ func keyNotExists(t *testing.T, key []byte, etcd *embed.Etcd) { require.Len(t, r.KVs, 0) } -func rangeMatches(t *testing.T, ranges stream.Ranges, etcd *embed.Etcd) { +func rangeMatches(t *testing.T, ranges streamhelper.Ranges, etcd *embed.Etcd) { r, err := etcd.Server.KV().Range(context.TODO(), ranges[0].StartKey, ranges[len(ranges)-1].EndKey, mvcc.RangeOptions{}) require.NoError(t, err) if len(r.KVs) != len(ranges) { @@ -133,33 +133,34 @@ func rangeIsEmpty(t *testing.T, prefix []byte, etcd *embed.Etcd) { func TestIntegration(t *testing.T) { etcd, cli := runEtcd(t) defer etcd.Server.Stop() - metaCli := stream.MetaDataClient{Client: cli} + metaCli := streamhelper.MetaDataClient{Client: cli} t.Run("TestBasic", func(t *testing.T) { testBasic(t, metaCli, etcd) }) t.Run("TestForwardProgress", func(t *testing.T) { testForwardProgress(t, metaCli, etcd) }) + t.Run("TestStreamListening", func(t *testing.T) { testStreamListening(t, streamhelper.TaskEventClient{MetaDataClient: metaCli}) }) } func TestChecking(t *testing.T) { noop, _ := storage.ParseBackend("noop://", nil) // The name must not contains slash. - _, err := stream.NewTask("/root"). + _, err := streamhelper.NewTask("/root"). WithRange([]byte("1"), []byte("2")). WithTableFilter("*.*"). ToStorage(noop). Check() require.ErrorIs(t, errors.Cause(err), berrors.ErrPiTRInvalidTaskInfo) // Must specify the external storage. - _, err = stream.NewTask("root"). + _, err = streamhelper.NewTask("root"). WithRange([]byte("1"), []byte("2")). WithTableFilter("*.*"). Check() require.ErrorIs(t, errors.Cause(err), berrors.ErrPiTRInvalidTaskInfo) // Must specift the table filter and range? - _, err = stream.NewTask("root"). + _, err = streamhelper.NewTask("root"). ToStorage(noop). Check() require.ErrorIs(t, errors.Cause(err), berrors.ErrPiTRInvalidTaskInfo) // Happy path. - _, err = stream.NewTask("root"). + _, err = streamhelper.NewTask("root"). WithRange([]byte("1"), []byte("2")). WithTableFilter("*.*"). ToStorage(noop). @@ -167,43 +168,43 @@ func TestChecking(t *testing.T) { require.NoError(t, err) } -func testBasic(t *testing.T, metaCli stream.MetaDataClient, etcd *embed.Etcd) { +func testBasic(t *testing.T, metaCli streamhelper.MetaDataClient, etcd *embed.Etcd) { ctx := context.Background() taskName := "two_tables" task := simpleTask(taskName, 2) taskData, err := task.PBInfo.Marshal() require.NoError(t, err) require.NoError(t, metaCli.PutTask(ctx, task)) - keyIs(t, []byte(stream.TaskOf(taskName)), taskData, etcd) - keyNotExists(t, []byte(stream.Pause(taskName)), etcd) - rangeMatches(t, []stream.Range{ - {StartKey: []byte(stream.RangeKeyOf(taskName, tablecodec.EncodeTablePrefix(1))), EndKey: tablecodec.EncodeTablePrefix(2)}, - {StartKey: []byte(stream.RangeKeyOf(taskName, tablecodec.EncodeTablePrefix(3))), EndKey: tablecodec.EncodeTablePrefix(4)}, + keyIs(t, []byte(streamhelper.TaskOf(taskName)), taskData, etcd) + keyNotExists(t, []byte(streamhelper.Pause(taskName)), etcd) + rangeMatches(t, []streamhelper.Range{ + {StartKey: []byte(streamhelper.RangeKeyOf(taskName, tablecodec.EncodeTablePrefix(1))), EndKey: tablecodec.EncodeTablePrefix(2)}, + {StartKey: []byte(streamhelper.RangeKeyOf(taskName, tablecodec.EncodeTablePrefix(3))), EndKey: tablecodec.EncodeTablePrefix(4)}, }, etcd) remoteTask, err := metaCli.GetTask(ctx, taskName) require.NoError(t, err) require.NoError(t, remoteTask.Pause(ctx)) - keyExists(t, []byte(stream.Pause(taskName)), etcd) + keyExists(t, []byte(streamhelper.Pause(taskName)), etcd) require.NoError(t, metaCli.PauseTask(ctx, taskName)) - keyExists(t, []byte(stream.Pause(taskName)), etcd) + keyExists(t, []byte(streamhelper.Pause(taskName)), etcd) paused, err := remoteTask.IsPaused(ctx) require.NoError(t, err) require.True(t, paused) require.NoError(t, metaCli.ResumeTask(ctx, taskName)) - keyNotExists(t, []byte(stream.Pause(taskName)), etcd) + keyNotExists(t, []byte(streamhelper.Pause(taskName)), etcd) require.NoError(t, metaCli.ResumeTask(ctx, taskName)) - keyNotExists(t, []byte(stream.Pause(taskName)), etcd) + keyNotExists(t, []byte(streamhelper.Pause(taskName)), etcd) paused, err = remoteTask.IsPaused(ctx) require.NoError(t, err) require.False(t, paused) require.NoError(t, metaCli.DeleteTask(ctx, taskName)) - keyNotExists(t, []byte(stream.TaskOf(taskName)), etcd) - rangeIsEmpty(t, []byte(stream.RangesOf(taskName)), etcd) + keyNotExists(t, []byte(streamhelper.TaskOf(taskName)), etcd) + rangeIsEmpty(t, []byte(streamhelper.RangesOf(taskName)), etcd) } -func testForwardProgress(t *testing.T, metaCli stream.MetaDataClient, etcd *embed.Etcd) { +func testForwardProgress(t *testing.T, metaCli streamhelper.MetaDataClient, etcd *embed.Etcd) { ctx := context.Background() taskName := "many_tables" taskInfo := simpleTask(taskName, 65) @@ -227,3 +228,34 @@ func testForwardProgress(t *testing.T, metaCli stream.MetaDataClient, etcd *embe require.NoError(t, err) require.Equal(t, store2Checkpoint, uint64(40)) } + +func testStreamListening(t *testing.T, metaCli streamhelper.TaskEventClient) { + ctx, cancel := context.WithCancel(context.Background()) + taskName := "simple" + taskInfo := simpleTask(taskName, 4) + + require.NoError(t, metaCli.PutTask(ctx, taskInfo)) + ch := make(chan streamhelper.TaskEvent, 1024) + require.NoError(t, metaCli.Begin(ctx, ch)) + require.NoError(t, metaCli.DeleteTask(ctx, taskName)) + + taskName2 := "simple2" + taskInfo2 := simpleTask(taskName2, 4) + require.NoError(t, metaCli.PutTask(ctx, taskInfo2)) + require.NoError(t, metaCli.DeleteTask(ctx, taskName2)) + first := <-ch + require.Equal(t, first.Type, streamhelper.EventAdd) + require.Equal(t, first.Name, taskName) + second := <-ch + require.Equal(t, second.Type, streamhelper.EventDel) + require.Equal(t, second.Name, taskName) + third := <-ch + require.Equal(t, third.Type, streamhelper.EventAdd) + require.Equal(t, third.Name, taskName2) + forth := <-ch + require.Equal(t, forth.Type, streamhelper.EventDel) + require.Equal(t, forth.Name, taskName2) + cancel() + _, ok := <-ch + require.False(t, ok) +} diff --git a/br/pkg/stream/models.go b/br/pkg/streamhelper/models.go similarity index 92% rename from br/pkg/stream/models.go rename to br/pkg/streamhelper/models.go index 7aee22de0c239..265669799a581 100644 --- a/br/pkg/stream/models.go +++ b/br/pkg/streamhelper/models.go @@ -1,5 +1,5 @@ // Copyright 2021 PingCAP, Inc. Licensed under Apache-2.0. -package stream +package streamhelper import ( "bytes" @@ -21,10 +21,13 @@ const ( streamKeyPrefix = "/tidb/br-stream" taskInfoPath = "/info" // nolint:deadcode,varcheck - taskCheckpointPath = "/checkpoint" - taskRangesPath = "/ranges" - taskPausePath = "/pause" - taskLastErrorPath = "/last-error" + taskCheckpointPath = "/checkpoint" + taskRangesPath = "/ranges" + taskPausePath = "/pause" + taskLastErrorPath = "/last-error" + checkpointTypeGlobal = "central_global" + checkpointTypeRegion = "region" + checkpointTypeStore = "store" ) var ( @@ -78,6 +81,11 @@ func CheckPointsOf(task string) string { return buf.String() } +// GlobalCheckpointOf returns the path to the "global" checkpoint of some task. +func GlobalCheckpointOf(task string) string { + return path.Join(streamKeyPrefix, taskCheckpointPath, task, checkpointTypeGlobal) +} + // CheckpointOf returns the checkpoint prefix of some store. // Normally it would be /checkpoint//. func CheckPointOf(task string, store uint64) string { diff --git a/br/pkg/stream/prefix_scanner.go b/br/pkg/streamhelper/prefix_scanner.go similarity index 99% rename from br/pkg/stream/prefix_scanner.go rename to br/pkg/streamhelper/prefix_scanner.go index 4700b26c5acd2..c06b3b9a26867 100644 --- a/br/pkg/stream/prefix_scanner.go +++ b/br/pkg/streamhelper/prefix_scanner.go @@ -1,5 +1,5 @@ // Copyright 2021 PingCAP, Inc. Licensed under Apache-2.0. -package stream +package streamhelper import ( "context" diff --git a/br/pkg/streamhelper/regioniter.go b/br/pkg/streamhelper/regioniter.go new file mode 100644 index 0000000000000..b2bfa0820316c --- /dev/null +++ b/br/pkg/streamhelper/regioniter.go @@ -0,0 +1,122 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. + +package streamhelper + +import ( + "bytes" + "context" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/metapb" + berrors "github.com/pingcap/tidb/br/pkg/errors" + "github.com/pingcap/tidb/br/pkg/redact" + "github.com/pingcap/tidb/br/pkg/utils" +) + +const ( + defaultPageSize = 2048 +) + +type RegionWithLeader struct { + Region *metapb.Region + Leader *metapb.Peer +} + +type RegionScanner interface { + // RegionScan gets a list of regions, starts from the region that contains key. + // Limit limits the maximum number of regions returned. + RegionScan(ctx context.Context, key, endKey []byte, limit int) ([]RegionWithLeader, error) +} + +type RegionIter struct { + cli RegionScanner + startKey, endKey []byte + currentStartKey []byte + // When the endKey become "", we cannot check whether the scan is done by + // comparing currentStartKey and endKey (because "" has different meaning in start key and end key). + // So set this to `ture` when endKey == "" and the scan is done. + infScanFinished bool + + // The max slice size returned by `Next`. + // This can be changed before calling `Next` each time, + // however no thread safety provided. + PageSize int +} + +// IterateRegion creates an iterater over the region range. +func IterateRegion(cli RegionScanner, startKey, endKey []byte) *RegionIter { + return &RegionIter{ + cli: cli, + startKey: startKey, + endKey: endKey, + currentStartKey: startKey, + PageSize: defaultPageSize, + } +} + +func CheckRegionConsistency(startKey, endKey []byte, regions []RegionWithLeader) error { + // current pd can't guarantee the consistency of returned regions + if len(regions) == 0 { + return errors.Annotatef(berrors.ErrPDBatchScanRegion, "scan region return empty result, startKey: %s, endKey: %s", + redact.Key(startKey), redact.Key(endKey)) + } + + if bytes.Compare(regions[0].Region.StartKey, startKey) > 0 { + return errors.Annotatef(berrors.ErrPDBatchScanRegion, "first region's startKey > startKey, startKey: %s, regionStartKey: %s", + redact.Key(startKey), redact.Key(regions[0].Region.StartKey)) + } else if len(regions[len(regions)-1].Region.EndKey) != 0 && bytes.Compare(regions[len(regions)-1].Region.EndKey, endKey) < 0 { + return errors.Annotatef(berrors.ErrPDBatchScanRegion, "last region's endKey < endKey, endKey: %s, regionEndKey: %s", + redact.Key(endKey), redact.Key(regions[len(regions)-1].Region.EndKey)) + } + + cur := regions[0] + for _, r := range regions[1:] { + if !bytes.Equal(cur.Region.EndKey, r.Region.StartKey) { + return errors.Annotatef(berrors.ErrPDBatchScanRegion, "region endKey not equal to next region startKey, endKey: %s, startKey: %s", + redact.Key(cur.Region.EndKey), redact.Key(r.Region.StartKey)) + } + cur = r + } + + return nil +} + +// Next get the next page of regions. +func (r *RegionIter) Next(ctx context.Context) ([]RegionWithLeader, error) { + var rs []RegionWithLeader + state := utils.InitialRetryState(30, 500*time.Millisecond, 500*time.Millisecond) + err := utils.WithRetry(ctx, func() error { + regions, err := r.cli.RegionScan(ctx, r.currentStartKey, r.endKey, r.PageSize) + if err != nil { + return err + } + if len(regions) > 0 { + endKey := regions[len(regions)-1].Region.GetEndKey() + if err := CheckRegionConsistency(r.currentStartKey, endKey, regions); err != nil { + return err + } + rs = regions + return nil + } + return CheckRegionConsistency(r.currentStartKey, r.endKey, regions) + }, &state) + if err != nil { + return nil, err + } + endKey := rs[len(rs)-1].Region.EndKey + // We have meet the last region. + if len(endKey) == 0 { + r.infScanFinished = true + } + r.currentStartKey = endKey + return rs, nil +} + +// Done checks whether the iteration is done. +func (r *RegionIter) Done() bool { + if len(r.endKey) == 0 { + return r.infScanFinished + } + return bytes.Compare(r.currentStartKey, r.endKey) >= 0 +} diff --git a/br/pkg/streamhelper/stream_listener.go b/br/pkg/streamhelper/stream_listener.go new file mode 100644 index 0000000000000..e48064613efdb --- /dev/null +++ b/br/pkg/streamhelper/stream_listener.go @@ -0,0 +1,170 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. + +package streamhelper + +import ( + "bytes" + "context" + "fmt" + "strings" + + "github.com/golang/protobuf/proto" + "github.com/pingcap/errors" + backuppb "github.com/pingcap/kvproto/pkg/brpb" + berrors "github.com/pingcap/tidb/br/pkg/errors" + clientv3 "go.etcd.io/etcd/client/v3" +) + +type EventType int + +const ( + EventAdd EventType = iota + EventDel + EventErr +) + +func (t EventType) String() string { + switch t { + case EventAdd: + return "Add" + case EventDel: + return "Del" + case EventErr: + return "Err" + } + return "Unknown" +} + +type TaskEvent struct { + Type EventType + Name string + Info *backuppb.StreamBackupTaskInfo + Err error +} + +func (t *TaskEvent) String() string { + if t.Err != nil { + return fmt.Sprintf("%s(%s, err = %s)", t.Type, t.Name, t.Err) + } + return fmt.Sprintf("%s(%s)", t.Type, t.Name) +} + +type TaskEventClient struct { + MetaDataClient +} + +func errorEvent(err error) TaskEvent { + return TaskEvent{ + Type: EventErr, + Err: err, + } +} + +func toTaskEvent(event *clientv3.Event) (TaskEvent, error) { + if !bytes.HasPrefix(event.Kv.Key, []byte(PrefixOfTask())) { + return TaskEvent{}, errors.Annotatef(berrors.ErrInvalidArgument, "the path isn't a task path (%s)", string(event.Kv.Key)) + } + + te := TaskEvent{} + te.Name = strings.TrimPrefix(string(event.Kv.Key), PrefixOfTask()) + if event.Type == clientv3.EventTypeDelete { + te.Type = EventDel + } else if event.Type == clientv3.EventTypePut { + te.Type = EventAdd + } else { + return TaskEvent{}, errors.Annotatef(berrors.ErrInvalidArgument, "event type is wrong (%s)", event.Type) + } + te.Info = new(backuppb.StreamBackupTaskInfo) + if err := proto.Unmarshal(event.Kv.Value, te.Info); err != nil { + return TaskEvent{}, err + } + return te, nil +} + +func eventFromWatch(resp clientv3.WatchResponse) ([]TaskEvent, error) { + result := make([]TaskEvent, 0, len(resp.Events)) + for _, event := range resp.Events { + te, err := toTaskEvent(event) + if err != nil { + te.Type = EventErr + te.Err = err + } + result = append(result, te) + } + return result, nil +} + +func (t TaskEventClient) startListen(ctx context.Context, rev int64, ch chan<- TaskEvent) { + c := t.Client.Watcher.Watch(ctx, PrefixOfTask(), clientv3.WithPrefix(), clientv3.WithRev(rev)) + handleResponse := func(resp clientv3.WatchResponse) bool { + events, err := eventFromWatch(resp) + if err != nil { + ch <- errorEvent(err) + return false + } + for _, event := range events { + ch <- event + } + return true + } + + go func() { + defer close(ch) + for { + select { + case resp, ok := <-c: + if !ok { + return + } + if !handleResponse(resp) { + return + } + case <-ctx.Done(): + // drain the remain event from channel. + for { + select { + case resp, ok := <-c: + if !ok { + return + } + if !handleResponse(resp) { + return + } + default: + return + } + } + } + } + }() +} + +func (t TaskEventClient) getFullTasksAsEvent(ctx context.Context) ([]TaskEvent, int64, error) { + tasks, rev, err := t.GetAllTasksWithRevision(ctx) + if err != nil { + return nil, 0, err + } + events := make([]TaskEvent, 0, len(tasks)) + for _, task := range tasks { + te := TaskEvent{ + Type: EventAdd, + Name: task.Info.Name, + Info: &task.Info, + } + events = append(events, te) + } + return events, rev, nil +} + +func (t TaskEventClient) Begin(ctx context.Context, ch chan<- TaskEvent) error { + initialTasks, rev, err := t.getFullTasksAsEvent(ctx) + if err != nil { + return err + } + // Note: maybe `go` here so we won't block? + for _, task := range initialTasks { + ch <- task + } + t.startListen(ctx, rev+1, ch) + return nil +} diff --git a/br/pkg/streamhelper/tsheap.go b/br/pkg/streamhelper/tsheap.go new file mode 100644 index 0000000000000..64669a151467a --- /dev/null +++ b/br/pkg/streamhelper/tsheap.go @@ -0,0 +1,216 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. + +package streamhelper + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/google/btree" + "github.com/pingcap/errors" + berrors "github.com/pingcap/tidb/br/pkg/errors" + "github.com/pingcap/tidb/br/pkg/logutil" + "github.com/pingcap/tidb/kv" + "github.com/tikv/client-go/v2/oracle" +) + +// CheckpointsCache is the heap-like cache for checkpoints. +// +// "Checkpoint" is the "Resolved TS" of some range. +// A resolved ts is a "watermark" for the system, which: +// - implies there won't be any transactions (in some range) commit with `commit_ts` smaller than this TS. +// - is monotonic increasing. +// A "checkpoint" is a "safe" Resolved TS, which: +// - is a TS *less than* the real resolved ts of now. +// - is based on range (it only promises there won't be new committed txns in the range). +// - the checkpoint of union of ranges is the minimal checkpoint of all ranges. +// As an example: +// +----------------------------------+ +// ^-----------^ (Checkpoint = 42) +// ^---------------^ (Checkpoint = 76) +// ^-----------------------^ (Checkpoint = min(42, 76) = 42) +// +// For calculating the global checkpoint, we can make a heap-like structure: +// Checkpoint Ranges +// 42 -> {[0, 8], [16, 100]} +// 1002 -> {[8, 16]} +// 1082 -> {[100, inf]} +// For now, the checkpoint of range [8, 16] and [100, inf] won't affect the global checkpoint +// directly, so we can try to advance only the ranges of {[0, 8], [16, 100]} (which's checkpoint is steal). +// Once them get advance, the global checkpoint would be advanced then, +// and we don't need to update all ranges (because some new ranges don't need to be advanced so quickly.) +type CheckpointsCache interface { + fmt.Stringer + // InsertRange inserts a range with specified TS to the cache. + InsertRange(ts uint64, rng kv.KeyRange) + // InsertRanges inserts a set of ranges that sharing checkpoint to the cache. + InsertRanges(rst RangesSharesTS) + // CheckpointTS returns the now global (union of all ranges) checkpoint of the cache. + CheckpointTS() uint64 + // PopRangesWithGapGT pops the ranges which's checkpoint is + PopRangesWithGapGT(d time.Duration) []*RangesSharesTS + // Check whether the ranges in the cache is integrate. + ConsistencyCheck() error + // Clear the cache. + Clear() +} + +// NoOPCheckpointCache is used when cache disabled. +type NoOPCheckpointCache struct{} + +func (NoOPCheckpointCache) InsertRange(ts uint64, rng kv.KeyRange) {} + +func (NoOPCheckpointCache) InsertRanges(rst RangesSharesTS) {} + +func (NoOPCheckpointCache) Clear() {} + +func (NoOPCheckpointCache) String() string { + return "NoOPCheckpointCache" +} + +func (NoOPCheckpointCache) CheckpointTS() uint64 { + panic("invalid state: NoOPCheckpointCache should never be used in advancing!") +} + +func (NoOPCheckpointCache) PopRangesWithGapGT(d time.Duration) []*RangesSharesTS { + panic("invalid state: NoOPCheckpointCache should never be used in advancing!") +} + +func (NoOPCheckpointCache) ConsistencyCheck() error { + return errors.Annotatef(berrors.ErrUnsupportedOperation, "invalid state: NoOPCheckpointCache should never be used in advancing!") +} + +// RangesSharesTS is a set of ranges shares the same timestamp. +type RangesSharesTS struct { + TS uint64 + Ranges []kv.KeyRange +} + +func (rst *RangesSharesTS) String() string { + // Make a more friendly string. + return fmt.Sprintf("@%sR%d", oracle.GetTimeFromTS(rst.TS).Format("0405"), len(rst.Ranges)) +} + +func (rst *RangesSharesTS) Less(other btree.Item) bool { + return rst.TS < other.(*RangesSharesTS).TS +} + +// Checkpoints is a heap that collects all checkpoints of +// regions, it supports query the latest checkpoint fast. +// This structure is thread safe. +type Checkpoints struct { + tree *btree.BTree + + mu sync.Mutex +} + +func NewCheckpoints() *Checkpoints { + return &Checkpoints{ + tree: btree.New(32), + } +} + +// String formats the slowest 5 ranges sharing TS to string. +func (h *Checkpoints) String() string { + h.mu.Lock() + defer h.mu.Unlock() + + b := new(strings.Builder) + count := 0 + total := h.tree.Len() + h.tree.Ascend(func(i btree.Item) bool { + rst := i.(*RangesSharesTS) + b.WriteString(rst.String()) + b.WriteString(";") + count++ + return count < 5 + }) + if total-count > 0 { + fmt.Fprintf(b, "O%d", total-count) + } + return b.String() +} + +// InsertRanges insert a RangesSharesTS directly to the tree. +func (h *Checkpoints) InsertRanges(r RangesSharesTS) { + h.mu.Lock() + defer h.mu.Unlock() + if items := h.tree.Get(&r); items != nil { + i := items.(*RangesSharesTS) + i.Ranges = append(i.Ranges, r.Ranges...) + } else { + h.tree.ReplaceOrInsert(&r) + } +} + +// InsertRange inserts the region and its TS into the region tree. +func (h *Checkpoints) InsertRange(ts uint64, rng kv.KeyRange) { + h.mu.Lock() + defer h.mu.Unlock() + r := h.tree.Get(&RangesSharesTS{TS: ts}) + if r == nil { + r = &RangesSharesTS{TS: ts} + h.tree.ReplaceOrInsert(r) + } + rr := r.(*RangesSharesTS) + rr.Ranges = append(rr.Ranges, rng) +} + +// Clear removes all records in the checkpoint cache. +func (h *Checkpoints) Clear() { + h.mu.Lock() + defer h.mu.Unlock() + h.tree.Clear(false) +} + +// PopRangesWithGapGT pops ranges with gap greater than the specified duration. +// NOTE: maybe make something like `DrainIterator` for better composing? +func (h *Checkpoints) PopRangesWithGapGT(d time.Duration) []*RangesSharesTS { + h.mu.Lock() + defer h.mu.Unlock() + result := []*RangesSharesTS{} + for { + item, ok := h.tree.Min().(*RangesSharesTS) + if !ok { + return result + } + if time.Since(oracle.GetTimeFromTS(item.TS)) >= d { + result = append(result, item) + h.tree.DeleteMin() + } else { + return result + } + } +} + +// CheckpointTS returns the cached checkpoint TS by the current state of the cache. +func (h *Checkpoints) CheckpointTS() uint64 { + h.mu.Lock() + defer h.mu.Unlock() + item, ok := h.tree.Min().(*RangesSharesTS) + if !ok { + return 0 + } + return item.TS +} + +// ConsistencyCheck checks whether the tree contains the full range of key space. +// TODO: add argument to it and check a sub range. +func (h *Checkpoints) ConsistencyCheck() error { + h.mu.Lock() + ranges := make([]kv.KeyRange, 0, 1024) + h.tree.Ascend(func(i btree.Item) bool { + ranges = append(ranges, i.(*RangesSharesTS).Ranges...) + return true + }) + h.mu.Unlock() + + r := CollapseRanges(len(ranges), func(i int) kv.KeyRange { return ranges[i] }) + if len(r) != 1 || len(r[0].StartKey) != 0 || len(r[0].EndKey) != 0 { + return errors.Annotatef(berrors.ErrPiTRMalformedMetadata, + "the region tree cannot cover the key space, collapsed: %s", logutil.StringifyKeys(r)) + } + return nil +} diff --git a/br/pkg/streamhelper/tsheap_test.go b/br/pkg/streamhelper/tsheap_test.go new file mode 100644 index 0000000000000..843dbf3f42f09 --- /dev/null +++ b/br/pkg/streamhelper/tsheap_test.go @@ -0,0 +1,161 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. +package streamhelper_test + +import ( + "math" + "testing" + + "github.com/pingcap/tidb/br/pkg/streamhelper" + "github.com/pingcap/tidb/kv" + "github.com/stretchr/testify/require" +) + +func TestInsert(t *testing.T) { + cases := []func(func(ts uint64, a, b string)){ + func(insert func(ts uint64, a, b string)) { + insert(1, "", "01") + insert(1, "01", "02") + insert(2, "02", "022") + insert(4, "022", "") + }, + func(insert func(ts uint64, a, b string)) { + insert(1, "", "01") + insert(2, "", "01") + insert(2, "011", "02") + insert(1, "", "") + insert(65, "03", "04") + }, + } + + for _, c := range cases { + cps := streamhelper.NewCheckpoints() + expected := map[uint64]*streamhelper.RangesSharesTS{} + checkpoint := uint64(math.MaxUint64) + insert := func(ts uint64, a, b string) { + cps.InsertRange(ts, kv.KeyRange{ + StartKey: []byte(a), + EndKey: []byte(b), + }) + i, ok := expected[ts] + if !ok { + expected[ts] = &streamhelper.RangesSharesTS{TS: ts, Ranges: []kv.KeyRange{{StartKey: []byte(a), EndKey: []byte(b)}}} + } else { + i.Ranges = append(i.Ranges, kv.KeyRange{StartKey: []byte(a), EndKey: []byte(b)}) + } + if ts < checkpoint { + checkpoint = ts + } + } + c(insert) + require.Equal(t, checkpoint, cps.CheckpointTS()) + rngs := cps.PopRangesWithGapGT(0) + for _, rng := range rngs { + other := expected[rng.TS] + require.Equal(t, other, rng) + } + } +} + +func TestMergeRanges(t *testing.T) { + r := func(a, b string) kv.KeyRange { + return kv.KeyRange{StartKey: []byte(a), EndKey: []byte(b)} + } + type Case struct { + expected []kv.KeyRange + parameter []kv.KeyRange + } + cases := []Case{ + { + parameter: []kv.KeyRange{r("01", "01111"), r("0111", "0112")}, + expected: []kv.KeyRange{r("01", "0112")}, + }, + { + parameter: []kv.KeyRange{r("01", "03"), r("02", "04")}, + expected: []kv.KeyRange{r("01", "04")}, + }, + { + parameter: []kv.KeyRange{r("04", "08"), r("09", "10")}, + expected: []kv.KeyRange{r("04", "08"), r("09", "10")}, + }, + { + parameter: []kv.KeyRange{r("01", "03"), r("02", "04"), r("05", "07"), r("08", "09")}, + expected: []kv.KeyRange{r("01", "04"), r("05", "07"), r("08", "09")}, + }, + { + parameter: []kv.KeyRange{r("01", "02"), r("012", "")}, + expected: []kv.KeyRange{r("01", "")}, + }, + { + parameter: []kv.KeyRange{r("", "01"), r("02", "03"), r("021", "")}, + expected: []kv.KeyRange{r("", "01"), r("02", "")}, + }, + { + parameter: []kv.KeyRange{r("", "01"), r("001", "")}, + expected: []kv.KeyRange{r("", "")}, + }, + { + parameter: []kv.KeyRange{r("", "01"), r("", ""), r("", "02")}, + expected: []kv.KeyRange{r("", "")}, + }, + { + parameter: []kv.KeyRange{r("", "01"), r("01", ""), r("", "02"), r("", "03"), r("01", "02")}, + expected: []kv.KeyRange{r("", "")}, + }, + } + + for i, c := range cases { + result := streamhelper.CollapseRanges(len(c.parameter), func(i int) kv.KeyRange { + return c.parameter[i] + }) + require.Equal(t, c.expected, result, "case = %d", i) + } + +} + +func TestInsertRanges(t *testing.T) { + r := func(a, b string) kv.KeyRange { + return kv.KeyRange{StartKey: []byte(a), EndKey: []byte(b)} + } + rs := func(ts uint64, ranges ...kv.KeyRange) streamhelper.RangesSharesTS { + return streamhelper.RangesSharesTS{TS: ts, Ranges: ranges} + } + + type Case struct { + Expected []streamhelper.RangesSharesTS + Parameters []streamhelper.RangesSharesTS + } + + cases := []Case{ + { + Parameters: []streamhelper.RangesSharesTS{ + rs(1, r("0", "1"), r("1", "2")), + rs(1, r("2", "3"), r("3", "4")), + }, + Expected: []streamhelper.RangesSharesTS{ + rs(1, r("0", "1"), r("1", "2"), r("2", "3"), r("3", "4")), + }, + }, + { + Parameters: []streamhelper.RangesSharesTS{ + rs(1, r("0", "1")), + rs(2, r("2", "3")), + rs(1, r("4", "5"), r("6", "7")), + }, + Expected: []streamhelper.RangesSharesTS{ + rs(1, r("0", "1"), r("4", "5"), r("6", "7")), + rs(2, r("2", "3")), + }, + }, + } + + for _, c := range cases { + theTree := streamhelper.NewCheckpoints() + for _, p := range c.Parameters { + theTree.InsertRanges(p) + } + ranges := theTree.PopRangesWithGapGT(0) + for i, rs := range ranges { + require.ElementsMatch(t, c.Expected[i].Ranges, rs.Ranges, "case = %#v", c) + } + } +} diff --git a/br/pkg/task/stream.go b/br/pkg/task/stream.go index dca6ac3b50ada..160aa5ad6712a 100644 --- a/br/pkg/task/stream.go +++ b/br/pkg/task/stream.go @@ -39,6 +39,8 @@ import ( "github.com/pingcap/tidb/br/pkg/restore" "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/br/pkg/stream" + "github.com/pingcap/tidb/br/pkg/streamhelper" + advancercfg "github.com/pingcap/tidb/br/pkg/streamhelper/config" "github.com/pingcap/tidb/br/pkg/summary" "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/kv" @@ -70,6 +72,7 @@ var ( StreamStatus = "log status" StreamTruncate = "log truncate" StreamMetadata = "log metadata" + StreamCtl = "log ctl" skipSummaryCommandList = map[string]struct{}{ StreamStatus: {}, @@ -90,6 +93,7 @@ var StreamCommandMap = map[string]func(c context.Context, g glue.Glue, cmdName s StreamStatus: RunStreamStatus, StreamTruncate: RunStreamTruncate, StreamMetadata: RunStreamMetadata, + StreamCtl: RunStreamAdvancer, } // StreamConfig specifies the configure about backup stream @@ -111,6 +115,9 @@ type StreamConfig struct { // Spec for the command `status`. JSONOutput bool `json:"json-output" toml:"json-output"` + + // Spec for the command `advancer`. + AdvancerCfg advancercfg.Config `json:"advancer-config" toml:"advancer-config"` } func (cfg *StreamConfig) makeStorage(ctx context.Context) (storage.ExternalStorage, error) { @@ -521,7 +528,7 @@ func RunStreamStart( return errors.Trace(err) } - cli := stream.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) + cli := streamhelper.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) // It supports single stream log task currently. if count, err := cli.GetTaskCount(ctx); err != nil { return errors.Trace(err) @@ -548,7 +555,7 @@ func RunStreamStart( return errors.Annotate(berrors.ErrInvalidArgument, "nothing need to observe") } - ti := stream.TaskInfo{ + ti := streamhelper.TaskInfo{ PBInfo: backuppb.StreamBackupTaskInfo{ Storage: streamMgr.bc.GetStorageBackend(), StartTs: cfg.StartTS, @@ -623,7 +630,7 @@ func RunStreamStop( } defer streamMgr.close() - cli := stream.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) + cli := streamhelper.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) // to add backoff ti, err := cli.GetTask(ctx, cfg.TaskName) if err != nil { @@ -673,7 +680,7 @@ func RunStreamPause( } defer streamMgr.close() - cli := stream.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) + cli := streamhelper.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) // to add backoff ti, isPaused, err := cli.GetTaskWithPauseStatus(ctx, cfg.TaskName) if err != nil { @@ -731,7 +738,7 @@ func RunStreamResume( } defer streamMgr.close() - cli := stream.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) + cli := streamhelper.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) // to add backoff ti, isPaused, err := cli.GetTaskWithPauseStatus(ctx, cfg.TaskName) if err != nil { @@ -776,6 +783,31 @@ func RunStreamResume( return nil } +func RunStreamAdvancer(c context.Context, g glue.Glue, cmdName string, cfg *StreamConfig) error { + ctx, cancel := context.WithCancel(c) + defer cancel() + mgr, err := NewMgr(ctx, g, cfg.PD, cfg.TLS, GetKeepalive(&cfg.Config), + cfg.CheckRequirements, false) + if err != nil { + return err + } + + etcdCLI, err := dialEtcdWithCfg(ctx, cfg.Config) + if err != nil { + return err + } + env := streamhelper.CliEnv(mgr.StoreManager, etcdCLI) + advancer := streamhelper.NewCheckpointAdvancer(env) + advancer.UpdateConfig(cfg.AdvancerCfg) + daemon := streamhelper.NewAdvancerDaemon(advancer, streamhelper.OwnerManagerForLogBackup(ctx, etcdCLI)) + loop, err := daemon.Begin(ctx) + if err != nil { + return err + } + loop() + return nil +} + func checkConfigForStatus(cfg *StreamConfig) error { if len(cfg.PD) == 0 { return errors.Annotatef(berrors.ErrInvalidArgument, @@ -793,7 +825,7 @@ func makeStatusController(ctx context.Context, cfg *StreamConfig, g glue.Glue) ( if err != nil { return nil, err } - cli := stream.NewMetaDataClient(etcdCLI) + cli := streamhelper.NewMetaDataClient(etcdCLI) var printer stream.TaskPrinter if !cfg.JSONOutput { printer = stream.PrintTaskByTable(console) diff --git a/br/pkg/utils/store_manager.go b/br/pkg/utils/store_manager.go new file mode 100644 index 0000000000000..db7381842e6b1 --- /dev/null +++ b/br/pkg/utils/store_manager.go @@ -0,0 +1,244 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. + +package utils + +import ( + "context" + "crypto/tls" + "os" + "sync" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + backuppb "github.com/pingcap/kvproto/pkg/brpb" + "github.com/pingcap/log" + berrors "github.com/pingcap/tidb/br/pkg/errors" + "github.com/pingcap/tidb/br/pkg/logutil" + pd "github.com/tikv/pd/client" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" +) + +const ( + dialTimeout = 30 * time.Second + resetRetryTimes = 3 +) + +// Pool is a lazy pool of gRPC channels. +// When `Get` called, it lazily allocates new connection if connection not full. +// If it's full, then it will return allocated channels round-robin. +type Pool struct { + mu sync.Mutex + + conns []*grpc.ClientConn + next int + cap int + newConn func(ctx context.Context) (*grpc.ClientConn, error) +} + +func (p *Pool) takeConns() (conns []*grpc.ClientConn) { + p.mu.Lock() + defer p.mu.Unlock() + p.conns, conns = nil, p.conns + p.next = 0 + return conns +} + +// Close closes the conn pool. +func (p *Pool) Close() { + for _, c := range p.takeConns() { + if err := c.Close(); err != nil { + log.Warn("failed to close clientConn", zap.String("target", c.Target()), zap.Error(err)) + } + } +} + +// Get tries to get an existing connection from the pool, or make a new one if the pool not full. +func (p *Pool) Get(ctx context.Context) (*grpc.ClientConn, error) { + p.mu.Lock() + defer p.mu.Unlock() + if len(p.conns) < p.cap { + c, err := p.newConn(ctx) + if err != nil { + return nil, err + } + p.conns = append(p.conns, c) + return c, nil + } + + conn := p.conns[p.next] + p.next = (p.next + 1) % p.cap + return conn, nil +} + +// NewConnPool creates a new Pool by the specified conn factory function and capacity. +func NewConnPool(capacity int, newConn func(ctx context.Context) (*grpc.ClientConn, error)) *Pool { + return &Pool{ + cap: capacity, + conns: make([]*grpc.ClientConn, 0, capacity), + newConn: newConn, + + mu: sync.Mutex{}, + } +} + +type StoreManager struct { + pdClient pd.Client + grpcClis struct { + mu sync.Mutex + clis map[uint64]*grpc.ClientConn + } + keepalive keepalive.ClientParameters + tlsConf *tls.Config +} + +// NewStoreManager create a new manager for gRPC connections to stores. +func NewStoreManager(pdCli pd.Client, kl keepalive.ClientParameters, tlsConf *tls.Config) *StoreManager { + return &StoreManager{ + pdClient: pdCli, + grpcClis: struct { + mu sync.Mutex + clis map[uint64]*grpc.ClientConn + }{clis: make(map[uint64]*grpc.ClientConn)}, + keepalive: kl, + tlsConf: tlsConf, + } +} + +func (mgr *StoreManager) PDClient() pd.Client { + return mgr.pdClient +} + +func (mgr *StoreManager) getGrpcConnLocked(ctx context.Context, storeID uint64) (*grpc.ClientConn, error) { + failpoint.Inject("hint-get-backup-client", func(v failpoint.Value) { + log.Info("failpoint hint-get-backup-client injected, "+ + "process will notify the shell.", zap.Uint64("store", storeID)) + if sigFile, ok := v.(string); ok { + file, err := os.Create(sigFile) + if err != nil { + log.Warn("failed to create file for notifying, skipping notify", zap.Error(err)) + } + if file != nil { + file.Close() + } + } + time.Sleep(3 * time.Second) + }) + store, err := mgr.pdClient.GetStore(ctx, storeID) + if err != nil { + return nil, errors.Trace(err) + } + opt := grpc.WithInsecure() + if mgr.tlsConf != nil { + opt = grpc.WithTransportCredentials(credentials.NewTLS(mgr.tlsConf)) + } + ctx, cancel := context.WithTimeout(ctx, dialTimeout) + bfConf := backoff.DefaultConfig + bfConf.MaxDelay = time.Second * 3 + addr := store.GetPeerAddress() + if addr == "" { + addr = store.GetAddress() + } + conn, err := grpc.DialContext( + ctx, + addr, + opt, + grpc.WithBlock(), + grpc.WithConnectParams(grpc.ConnectParams{Backoff: bfConf}), + grpc.WithKeepaliveParams(mgr.keepalive), + ) + cancel() + if err != nil { + return nil, berrors.ErrFailedToConnect.Wrap(err).GenWithStack("failed to make connection to store %d", storeID) + } + return conn, nil +} + +func (mgr *StoreManager) WithConn(ctx context.Context, storeID uint64, f func(*grpc.ClientConn)) error { + if ctx.Err() != nil { + return errors.Trace(ctx.Err()) + } + + mgr.grpcClis.mu.Lock() + defer mgr.grpcClis.mu.Unlock() + + if conn, ok := mgr.grpcClis.clis[storeID]; ok { + // Find a cached backup client. + f(conn) + return nil + } + + conn, err := mgr.getGrpcConnLocked(ctx, storeID) + if err != nil { + return errors.Trace(err) + } + // Cache the conn. + mgr.grpcClis.clis[storeID] = conn + f(conn) + return nil +} + +// ResetBackupClient reset the connection for backup client. +func (mgr *StoreManager) ResetBackupClient(ctx context.Context, storeID uint64) (backuppb.BackupClient, error) { + if ctx.Err() != nil { + return nil, errors.Trace(ctx.Err()) + } + + mgr.grpcClis.mu.Lock() + defer mgr.grpcClis.mu.Unlock() + + if conn, ok := mgr.grpcClis.clis[storeID]; ok { + // Find a cached backup client. + log.Info("Reset backup client", zap.Uint64("storeID", storeID)) + err := conn.Close() + if err != nil { + log.Warn("close backup connection failed, ignore it", zap.Uint64("storeID", storeID)) + } + delete(mgr.grpcClis.clis, storeID) + } + var ( + conn *grpc.ClientConn + err error + ) + for retry := 0; retry < resetRetryTimes; retry++ { + conn, err = mgr.getGrpcConnLocked(ctx, storeID) + if err != nil { + log.Warn("failed to reset grpc connection, retry it", + zap.Int("retry time", retry), logutil.ShortError(err)) + time.Sleep(time.Duration(retry+3) * time.Second) + continue + } + mgr.grpcClis.clis[storeID] = conn + break + } + if err != nil { + return nil, errors.Trace(err) + } + return backuppb.NewBackupClient(conn), nil +} + +// Close closes all client in Mgr. +func (mgr *StoreManager) Close() { + if mgr == nil { + return + } + mgr.grpcClis.mu.Lock() + for _, cli := range mgr.grpcClis.clis { + err := cli.Close() + if err != nil { + log.Error("fail to close Mgr", zap.Error(err)) + } + } + mgr.grpcClis.mu.Unlock() +} + +func (mgr *StoreManager) TLSConfig() *tls.Config { + if mgr == nil { + return nil + } + return mgr.tlsConf +} diff --git a/br/pkg/utils/worker.go b/br/pkg/utils/worker.go index 773cfd41a64da..cf80770d0ae67 100644 --- a/br/pkg/utils/worker.go +++ b/br/pkg/utils/worker.go @@ -3,7 +3,10 @@ package utils import ( + "github.com/pingcap/errors" "github.com/pingcap/log" + berrors "github.com/pingcap/tidb/br/pkg/errors" + "github.com/pingcap/tidb/br/pkg/logutil" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) @@ -107,3 +110,23 @@ func (pool *WorkerPool) RecycleWorker(worker *Worker) { func (pool *WorkerPool) HasWorker() bool { return pool.IdleCount() > 0 } + +// PanicToErr recovers when the execution get panicked, and set the error provided by the arg. +// generally, this would be used with named return value and `defer`, like: +// +// func foo() (err error) { +// defer utils.PanicToErr(&err) +// return maybePanic() +// } +// +// Before using this, there are some hints for reducing resource leakage or bugs: +// - If any of clean work (by `defer`) relies on the error (say, when error happens, rollback some operations.), please +// place `defer this` AFTER that. +// - All resources allocated should be freed by the `defer` syntax, or when panicking, they may not be recycled. +func PanicToErr(err *error) { + item := recover() + if item != nil { + *err = errors.Annotatef(berrors.ErrUnknown, "panicked when executing, message: %v", item) + log.Warn("checkpoint advancer panicked, recovering", zap.StackSkip("stack", 1), logutil.ShortError(*err)) + } +} diff --git a/config/config.go b/config/config.go index 1070e6847f2ba..d5bca9f1c2692 100644 --- a/config/config.go +++ b/config/config.go @@ -32,6 +32,7 @@ import ( "github.com/BurntSushi/toml" "github.com/pingcap/errors" zaplog "github.com/pingcap/log" + logbackupconf "github.com/pingcap/tidb/br/pkg/streamhelper/config" "github.com/pingcap/tidb/parser/terror" typejson "github.com/pingcap/tidb/types/json" "github.com/pingcap/tidb/util/logutil" @@ -256,8 +257,10 @@ type Config struct { // BallastObjectSize set the initial size of the ballast object, the unit is byte. BallastObjectSize int `toml:"ballast-object-size" json:"ballast-object-size"` // EnableGlobalKill indicates whether to enable global kill. - EnableGlobalKill bool `toml:"enable-global-kill" json:"enable-global-kill"` TrxSummary TrxSummary `toml:"transaction-summary" json:"transaction-summary"` + EnableGlobalKill bool `toml:"enable-global-kill" json:"enable-global-kill"` + // LogBackup controls the log backup related items. + LogBackup LogBackup `toml:"log-backup" json:"log-backup"` // The following items are deprecated. We need to keep them here temporarily // to support the upgrade process. They can be removed in future. @@ -416,6 +419,13 @@ func (b *AtomicBool) UnmarshalText(text []byte) error { return nil } +// LogBackup is the config for log backup service. +// For now, it includes the embed advancer. +type LogBackup struct { + Advancer logbackupconf.Config `toml:"advancer" json:"advancer"` + Enabled bool `toml:"enabled" json:"enabled"` +} + // Log is the log section of config. type Log struct { // Log level. @@ -942,6 +952,10 @@ var defaultConf = Config{ NewCollationsEnabledOnFirstBootstrap: true, EnableGlobalKill: true, TrxSummary: DefaultTrxSummary(), + LogBackup: LogBackup{ + Advancer: logbackupconf.Default(), + Enabled: false, + }, } var ( diff --git a/domain/domain.go b/domain/domain.go index 8b06ca3c736d0..0a83758aae4f4 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/bindinfo" + "github.com/pingcap/tidb/br/pkg/streamhelper" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" ddlutil "github.com/pingcap/tidb/ddl/util" @@ -92,6 +93,7 @@ type Domain struct { indexUsageSyncLease time.Duration dumpFileGcChecker *dumpFileGcChecker expiredTimeStamp4PC types.Time + logBackupAdvancer *streamhelper.AdvancerDaemon serverID uint64 serverIDSession *concurrency.Session @@ -889,10 +891,33 @@ func (do *Domain) Init(ddlLease time.Duration, sysExecutorFactory func(*Domain) do.wg.Add(1) go do.topologySyncerKeeper() } + err = do.initLogBackup(ctx, pdClient) + if err != nil { + return err + } return nil } +func (do *Domain) initLogBackup(ctx context.Context, pdClient pd.Client) error { + cfg := config.GetGlobalConfig() + if cfg.LogBackup.Enabled { + env, err := streamhelper.TiDBEnv(pdClient, do.etcdClient, cfg) + if err != nil { + return err + } + adv := streamhelper.NewCheckpointAdvancer(env) + adv.UpdateConfig(cfg.LogBackup.Advancer) + do.logBackupAdvancer = streamhelper.NewAdvancerDaemon(adv, streamhelper.OwnerManagerForLogBackup(ctx, do.etcdClient)) + loop, err := do.logBackupAdvancer.Begin(ctx) + if err != nil { + return err + } + do.wg.Run(loop) + } + return nil +} + type sessionPool struct { resources chan pools.Resource factory pools.Factory diff --git a/go.mod b/go.mod index 2d67c63904fc3..8cce36cca8da7 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 - github.com/pingcap/kvproto v0.0.0-20220705053936-aa9c2d20cd2a + github.com/pingcap/kvproto v0.0.0-20220705090230-a5d4ffd2ba33 github.com/pingcap/log v1.1.0 github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e diff --git a/go.sum b/go.sum index f4e4fb8b40f4f..53f2f6f7ec9ed 100644 --- a/go.sum +++ b/go.sum @@ -665,8 +665,8 @@ github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17Xtb github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20220302110454-c696585a961b/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20220525022339-6aaebf466305/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20220705053936-aa9c2d20cd2a h1:nP2wmyw9JTRsk5rm+tZtfAso6c/1FvuaFNbXTaYz3FE= -github.com/pingcap/kvproto v0.0.0-20220705053936-aa9c2d20cd2a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20220705090230-a5d4ffd2ba33 h1:VKMmvYhtG28j1sCCBdq4s+V9UOYqNgQ6CQviQwOgTeg= +github.com/pingcap/kvproto v0.0.0-20220705090230-a5d4ffd2ba33/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= diff --git a/metrics/log_backup.go b/metrics/log_backup.go new file mode 100644 index 0000000000000..b477f447c2dbb --- /dev/null +++ b/metrics/log_backup.go @@ -0,0 +1,51 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +// log backup metrics. +// see the `Help` field for details. +var ( + LastCheckpoint = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "log_backup", + Name: "last_checkpoint", + Help: "The last global checkpoint of log backup.", + }, []string{"task"}) + AdvancerOwner = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "log_backup", + Name: "advancer_owner", + Help: "If the node is the owner of advancers, set this to `1`, otherwise `0`.", + ConstLabels: map[string]string{}, + }) + AdvancerTickDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "log_backup", + Name: "advancer_tick_duration_sec", + Help: "The time cost of each step during advancer ticking.", + Buckets: prometheus.ExponentialBuckets(0.01, 3.0, 8), + }, []string{"step"}) + GetCheckpointBatchSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "log_backup", + Name: "advancer_batch_size", + Help: "The batch size of scanning region or get region checkpoint.", + Buckets: prometheus.ExponentialBuckets(1, 2.0, 12), + }, []string{"type"}) +) diff --git a/metrics/metrics.go b/metrics/metrics.go index 19809bd9c85d2..4011e587cec71 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -191,6 +191,10 @@ func RegisterMetrics() { prometheus.MustRegister(StatsHealthyGauge) prometheus.MustRegister(TxnStatusEnteringCounter) prometheus.MustRegister(TxnDurationHistogram) + prometheus.MustRegister(LastCheckpoint) + prometheus.MustRegister(AdvancerOwner) + prometheus.MustRegister(AdvancerTickDuration) + prometheus.MustRegister(GetCheckpointBatchSize) tikvmetrics.InitMetrics(TiDB, TiKVClient) tikvmetrics.RegisterMetrics() From dced89a231a87216e5cd1c2ca56296ee60f0b64c Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 12 Jul 2022 21:27:05 -0600 Subject: [PATCH 02/27] planner/core, planner, sessionctx/variable: remove more skipInit (#35992) ref pingcap/tidb#35051 --- planner/core/common_plans.go | 24 +++-------------- planner/optimize.go | 12 ++------- sessionctx/variable/noop.go | 2 +- sessionctx/variable/sysvar.go | 42 +++++++++++++----------------- sessionctx/variable/sysvar_test.go | 30 +++++---------------- 5 files changed, 32 insertions(+), 78 deletions(-) diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 93f7a9af2a51b..89873ff086650 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -35,7 +35,6 @@ import ( "github.com/pingcap/tidb/privilege" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" - "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" @@ -316,12 +315,6 @@ func (e *Execute) checkPreparedPriv(ctx context.Context, sctx sessionctx.Context return err } -func (e *Execute) setFoundInPlanCache(sctx sessionctx.Context, opt bool) error { - vars := sctx.GetSessionVars() - err := vars.SetSystemVar(variable.TiDBFoundInPlanCache, variable.BoolToOnOff(opt)) - return err -} - // GetBindSQL4PlanCache used to get the bindSQL for plan cache to build the plan cache key. func GetBindSQL4PlanCache(sctx sessionctx.Context, preparedStmt *CachedPrepareStmt) string { useBinding := sctx.GetSessionVars().UsePlanBaselines @@ -418,10 +411,7 @@ func (e *Execute) getPhysicalPlan(ctx context.Context, sctx sessionctx.Context, } else { planCacheCounter.Inc() } - err = e.setFoundInPlanCache(sctx, true) - if err != nil { - return err - } + sessVars.FoundInPlanCache = true e.names = names e.Plan = plan stmtCtx.PointExec = true @@ -460,17 +450,11 @@ func (e *Execute) getPhysicalPlan(ctx context.Context, sctx sessionctx.Context, logutil.BgLogger().Debug("rebuild range failed", zap.Error(err)) goto REBUILD } - err = e.setFoundInPlanCache(sctx, true) - if err != nil { - return err - } + sessVars.FoundInPlanCache = true if len(bindSQL) > 0 { // When the `len(bindSQL) > 0`, it means we use the binding. // So we need to record this. - err = sessVars.SetSystemVar(variable.TiDBFoundInBinding, variable.BoolToOnOff(true)) - if err != nil { - return err - } + sessVars.FoundInBinding = true } if metrics.ResettablePlanCacheCounterFortTest { metrics.PlanCacheCounter.WithLabelValues("prepare").Inc() @@ -534,7 +518,7 @@ REBUILD: sctx.PreparedPlanCache().Put(cacheKey, []*PlanCacheValue{cached}) } } - err = e.setFoundInPlanCache(sctx, false) + sessVars.FoundInPlanCache = false return err } diff --git a/planner/optimize.go b/planner/optimize.go index b6e2d84781265..aa3cfe4a6f1c4 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -186,9 +186,8 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in for _, warn := range warns { sessVars.StmtCtx.AppendWarning(warn) } - if err := setFoundInBinding(sctx, true, chosenBinding.BindSQL); err != nil { - logutil.BgLogger().Warn("set tidb_found_in_binding failed", zap.Error(err)) - } + sessVars.StmtCtx.BindSQL = chosenBinding.BindSQL + sessVars.FoundInBinding = true if sessVars.StmtCtx.InVerboseExplain { sessVars.StmtCtx.AppendNote(errors.Errorf("Using the bindSQL: %v", chosenBinding.BindSQL)) } @@ -664,13 +663,6 @@ func handleStmtHints(hints []*ast.TableOptimizerHint) (stmtHints stmtctx.StmtHin return } -func setFoundInBinding(sctx sessionctx.Context, opt bool, bindSQL string) error { - vars := sctx.GetSessionVars() - vars.StmtCtx.BindSQL = bindSQL - err := vars.SetSystemVar(variable.TiDBFoundInBinding, variable.BoolToOnOff(opt)) - return err -} - func init() { plannercore.OptimizeAstNode = Optimize plannercore.IsReadOnly = IsReadOnly diff --git a/sessionctx/variable/noop.go b/sessionctx/variable/noop.go index 6eb70beabfc99..30e72ad4bfb5c 100644 --- a/sessionctx/variable/noop.go +++ b/sessionctx/variable/noop.go @@ -155,7 +155,7 @@ var noopSysVars = []*SysVar{ {Scope: ScopeNone, Name: "innodb_buffer_pool_instances", Value: "8"}, {Scope: ScopeGlobal | ScopeSession, Name: "max_length_for_sort_data", Value: "1024", IsHintUpdatable: true}, {Scope: ScopeNone, Name: CharacterSetSystem, Value: "utf8"}, - {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetFilesystem, Value: "binary", skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetFilesystem, Value: "binary", Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCharacterSet(normalizedValue, CharacterSetFilesystem) }}, {Scope: ScopeGlobal, Name: InnodbOptimizeFullTextOnly, Value: "0"}, diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 4af4de4e12837..98c217cf3cf5a 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -71,7 +71,7 @@ var defaultSysVars = []*SysVar{ {Scope: ScopeNone, Name: TiDBAllowFunctionForExpressionIndex, ReadOnly: true, Value: collectAllowFuncName4ExpressionIndex()}, /* The system variables below have SESSION scope */ - {Scope: ScopeSession, Name: Timestamp, Value: DefTimestamp, skipInit: true, MinValue: 0, MaxValue: 2147483647, Type: TypeFloat, GetSession: func(s *SessionVars) (string, error) { + {Scope: ScopeSession, Name: Timestamp, Value: DefTimestamp, MinValue: 0, MaxValue: 2147483647, Type: TypeFloat, GetSession: func(s *SessionVars) (string, error) { if timestamp, ok := s.systems[Timestamp]; ok && timestamp != DefTimestamp { return timestamp, nil } @@ -81,18 +81,18 @@ var defaultSysVars = []*SysVar{ timestamp, ok := s.systems[Timestamp] return timestamp, ok && timestamp != DefTimestamp, nil }}, - {Scope: ScopeSession, Name: WarningCount, Value: "0", ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + {Scope: ScopeSession, Name: WarningCount, Value: "0", ReadOnly: true, GetSession: func(s *SessionVars) (string, error) { return strconv.Itoa(s.SysWarningCount), nil }}, - {Scope: ScopeSession, Name: ErrorCount, Value: "0", ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + {Scope: ScopeSession, Name: ErrorCount, Value: "0", ReadOnly: true, GetSession: func(s *SessionVars) (string, error) { return strconv.Itoa(int(s.SysErrorCount)), nil }}, - {Scope: ScopeSession, Name: LastInsertID, Value: "", skipInit: true, Type: TypeInt, AllowEmpty: true, MinValue: 0, MaxValue: math.MaxInt64, GetSession: func(s *SessionVars) (string, error) { + {Scope: ScopeSession, Name: LastInsertID, Value: "", Type: TypeInt, AllowEmpty: true, MinValue: 0, MaxValue: math.MaxInt64, GetSession: func(s *SessionVars) (string, error) { return strconv.FormatUint(s.StmtCtx.PrevLastInsertID, 10), nil }, GetStateValue: func(s *SessionVars) (string, bool, error) { return "", false, nil }}, - {Scope: ScopeSession, Name: Identity, Value: "", skipInit: true, Type: TypeInt, AllowEmpty: true, MinValue: 0, MaxValue: math.MaxInt64, GetSession: func(s *SessionVars) (string, error) { + {Scope: ScopeSession, Name: Identity, Value: "", Type: TypeInt, AllowEmpty: true, MinValue: 0, MaxValue: math.MaxInt64, GetSession: func(s *SessionVars) (string, error) { return strconv.FormatUint(s.StmtCtx.PrevLastInsertID, 10), nil }, GetStateValue: func(s *SessionVars) (string, bool, error) { return "", false, nil @@ -163,7 +163,7 @@ var defaultSysVars = []*SysVar{ s.AllowWriteRowID = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBChecksumTableConcurrency, skipInit: true, Value: strconv.Itoa(DefChecksumTableConcurrency), Type: TypeInt, MinValue: 1, MaxValue: MaxConfigurableConcurrency}, + {Scope: ScopeSession, Name: TiDBChecksumTableConcurrency, Value: strconv.Itoa(DefChecksumTableConcurrency), Type: TypeInt, MinValue: 1, MaxValue: MaxConfigurableConcurrency}, {Scope: ScopeSession, Name: TiDBBatchInsert, Value: BoolToOnOff(DefBatchInsert), Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.BatchInsert = TiDBOptOn(val) return nil @@ -176,13 +176,13 @@ var defaultSysVars = []*SysVar{ s.BatchCommit = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBCurrentTS, Value: strconv.Itoa(DefCurretTS), Type: TypeInt, AllowEmpty: true, MinValue: 0, MaxValue: math.MaxInt64, ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + {Scope: ScopeSession, Name: TiDBCurrentTS, Value: strconv.Itoa(DefCurretTS), Type: TypeInt, AllowEmpty: true, MinValue: 0, MaxValue: math.MaxInt64, ReadOnly: true, GetSession: func(s *SessionVars) (string, error) { return strconv.FormatUint(s.TxnCtx.StartTS, 10), nil }}, - {Scope: ScopeSession, Name: TiDBLastTxnInfo, Value: strconv.Itoa(DefCurretTS), ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + {Scope: ScopeSession, Name: TiDBLastTxnInfo, Value: strconv.Itoa(DefCurretTS), ReadOnly: true, GetSession: func(s *SessionVars) (string, error) { return s.LastTxnInfo, nil }}, - {Scope: ScopeSession, Name: TiDBLastQueryInfo, Value: strconv.Itoa(DefCurretTS), ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + {Scope: ScopeSession, Name: TiDBLastQueryInfo, Value: strconv.Itoa(DefCurretTS), ReadOnly: true, GetSession: func(s *SessionVars) (string, error) { info, err := json.Marshal(s.LastQueryInfo) if err != nil { return "", err @@ -279,16 +279,10 @@ var defaultSysVars = []*SysVar{ s.MetricSchemaRangeDuration = TidbOptInt64(val, DefTiDBMetricSchemaRangeDuration) return nil }}, - {Scope: ScopeSession, Name: TiDBFoundInPlanCache, Value: BoolToOnOff(DefTiDBFoundInPlanCache), Type: TypeBool, ReadOnly: true, skipInit: true, SetSession: func(s *SessionVars, val string) error { - s.FoundInPlanCache = TiDBOptOn(val) - return nil - }, GetSession: func(s *SessionVars) (string, error) { + {Scope: ScopeSession, Name: TiDBFoundInPlanCache, Value: BoolToOnOff(DefTiDBFoundInPlanCache), Type: TypeBool, ReadOnly: true, GetSession: func(s *SessionVars) (string, error) { return BoolToOnOff(s.PrevFoundInPlanCache), nil }}, - {Scope: ScopeSession, Name: TiDBFoundInBinding, Value: BoolToOnOff(DefTiDBFoundInBinding), Type: TypeBool, ReadOnly: true, skipInit: true, SetSession: func(s *SessionVars, val string) error { - s.FoundInBinding = TiDBOptOn(val) - return nil - }, GetSession: func(s *SessionVars) (string, error) { + {Scope: ScopeSession, Name: TiDBFoundInBinding, Value: BoolToOnOff(DefTiDBFoundInBinding), Type: TypeBool, ReadOnly: true, GetSession: func(s *SessionVars) (string, error) { return BoolToOnOff(s.PrevFoundInBinding), nil }}, {Scope: ScopeSession, Name: RandSeed1, Type: TypeInt, Value: "0", skipInit: true, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { @@ -316,7 +310,7 @@ var defaultSysVars = []*SysVar{ return nil }, }, - {Scope: ScopeSession, Name: TiDBLastDDLInfo, Value: strconv.Itoa(DefCurretTS), ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + {Scope: ScopeSession, Name: TiDBLastDDLInfo, Value: strconv.Itoa(DefCurretTS), ReadOnly: true, GetSession: func(s *SessionVars) (string, error) { info, err := json.Marshal(s.LastDDLInfo) if err != nil { return "", err @@ -867,7 +861,7 @@ var defaultSysVars = []*SysVar{ } return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: SQLLogBin, Value: On, Type: TypeBool, skipInit: true}, + {Scope: ScopeGlobal | ScopeSession, Name: SQLLogBin, Value: On, Type: TypeBool}, {Scope: ScopeGlobal | ScopeSession, Name: TimeZone, Value: "SYSTEM", IsHintUpdatable: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { if strings.EqualFold(normalizedValue, "SYSTEM") { return "SYSTEM", nil @@ -882,7 +876,7 @@ var defaultSysVars = []*SysVar{ s.TimeZone = tz return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: ForeignKeyChecks, Value: Off, Type: TypeBool, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: ForeignKeyChecks, Value: Off, Type: TypeBool, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { if TiDBOptOn(normalizedValue) { // TiDB does not yet support foreign keys. // Return the original value in the warning, so that users are not confused. @@ -911,10 +905,10 @@ var defaultSysVars = []*SysVar{ s.AutoIncrementOffset = tidbOptPositiveInt32(val, DefAutoIncrementOffset) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetClient, Value: mysql.DefaultCharset, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetClient, Value: mysql.DefaultCharset, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCharacterSet(normalizedValue, CharacterSetClient) }}, - {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetResults, Value: mysql.DefaultCharset, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetResults, Value: mysql.DefaultCharset, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { if normalizedValue == "" { return normalizedValue, nil } @@ -960,7 +954,7 @@ var defaultSysVars = []*SysVar{ s.LockWaitTimeout = lockWaitSec * 1000 return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: GroupConcatMaxLen, Value: "1024", IsHintUpdatable: true, skipInit: true, Type: TypeUnsigned, MinValue: 4, MaxValue: math.MaxUint64, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: GroupConcatMaxLen, Value: "1024", IsHintUpdatable: true, Type: TypeUnsigned, MinValue: 4, MaxValue: math.MaxUint64, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { // https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_group_concat_max_len // Minimum Value 4 // Maximum Value (64-bit platforms) 18446744073709551615 @@ -1047,7 +1041,7 @@ var defaultSysVars = []*SysVar{ s.BroadcastJoinThresholdSize = TidbOptInt64(val, DefBroadcastJoinThresholdSize) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBBuildStatsConcurrency, skipInit: true, Value: strconv.Itoa(DefBuildStatsConcurrency), Type: TypeInt, MinValue: 1, MaxValue: MaxConfigurableConcurrency}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBBuildStatsConcurrency, Value: strconv.Itoa(DefBuildStatsConcurrency), Type: TypeInt, MinValue: 1, MaxValue: MaxConfigurableConcurrency}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptCartesianBCJ, Value: strconv.Itoa(DefOptCartesianBCJ), Type: TypeInt, MinValue: 0, MaxValue: 2, SetSession: func(s *SessionVars, val string) error { s.AllowCartesianBCJ = TidbOptInt(val, DefOptCartesianBCJ) return nil diff --git a/sessionctx/variable/sysvar_test.go b/sessionctx/variable/sysvar_test.go index 6d94cb81e8ac0..1146732e6030c 100644 --- a/sessionctx/variable/sysvar_test.go +++ b/sessionctx/variable/sysvar_test.go @@ -703,33 +703,27 @@ func TestSkipInitIsUsed(t *testing.T) { // skipInit only ever applied to session scope, so if anyone is setting it on // a variable without session, that doesn't make sense. require.True(t, sv.HasSessionScope(), fmt.Sprintf("skipInit has no effect on a variable without session scope: %s", sv.Name)) + // Since SetSession is the "init function" there is no init function to skip. + require.NotNil(t, sv.SetSession, fmt.Sprintf("skipInit has no effect on variables without an init (setsession) func: %s", sv.Name)) + // Skipinit has no use on noop funcs, since noop funcs always skipinit. + require.False(t, sv.IsNoop, fmt.Sprintf("skipInit has no effect on noop variables: %s", sv.Name)) + // Many of these variables might allow skipInit to be removed, // they need to be checked first. The purpose of this test is to make // sure we don't introduce any new variables with skipInit, which seems // to be a problem. switch sv.Name { - case Timestamp, - WarningCount, - ErrorCount, - LastInsertID, - Identity, - TiDBTxnScope, + case TiDBTxnScope, TiDBSnapshot, TiDBOptDistinctAggPushDown, TiDBOptWriteRowID, - TiDBChecksumTableConcurrency, TiDBBatchInsert, TiDBBatchDelete, TiDBBatchCommit, - TiDBCurrentTS, - TiDBLastTxnInfo, - TiDBLastQueryInfo, TiDBEnableChunkRPC, TxnIsolationOneShot, TiDBOptimizerSelectivityLevel, TiDBOptimizerEnableOuterJoinReorder, - TiDBLogFileMaxDays, - TiDBConfig, TiDBDDLReorgPriority, TiDBSlowQueryFile, TiDBWaitSplitRegionFinish, @@ -738,27 +732,17 @@ func TestSkipInitIsUsed(t *testing.T) { TiDBAllowRemoveAutoInc, TiDBMetricSchemaStep, TiDBMetricSchemaRangeDuration, - TiDBFoundInPlanCache, - TiDBFoundInBinding, RandSeed1, RandSeed2, - TiDBLastDDLInfo, - SQLLogBin, - ForeignKeyChecks, CollationDatabase, - CharacterSetClient, - CharacterSetResults, CollationConnection, CharsetDatabase, - GroupConcatMaxLen, CharacterSetConnection, CharacterSetServer, - TiDBBuildStatsConcurrency, TiDBOptTiFlashConcurrencyFactor, TiDBOptSeekFactor, TiDBOptJoinReorderThreshold, - TiDBStatsLoadSyncWait, - CharacterSetFilesystem: + TiDBStatsLoadSyncWait: continue } require.Equal(t, false, sv.skipInit, fmt.Sprintf("skipInit should not be set on new system variables. variable %s is in violation", sv.Name)) From da1b82aae7aef8224c9e2708fd99da068c8d575a Mon Sep 17 00:00:00 2001 From: Song Gao Date: Wed, 13 Jul 2022 11:57:06 +0800 Subject: [PATCH 03/27] statistics: support tracking topn in statsCache (#35510) ref pingcap/tidb#34052 --- statistics/cmsketch.go | 3 + statistics/handle/lru_cache.go | 17 ++-- statistics/handle/lru_cache_test.go | 122 ++++++++++++++++++++-------- statistics/histogram.go | 118 ++++++++++++++++++++++++--- statistics/table.go | 37 ++++++++- 5 files changed, 243 insertions(+), 54 deletions(-) diff --git a/statistics/cmsketch.go b/statistics/cmsketch.go index 9e96dd0f801ed..44ce8d224e7bf 100644 --- a/statistics/cmsketch.go +++ b/statistics/cmsketch.go @@ -169,6 +169,9 @@ func calculateDefaultVal(helper *topNHelper, estimateNDV, scaleRatio, rowCount u // data are not tracked because size of CMSketch.topN take little influence // We ignore the size of other metadata in CMSketch. func (c *CMSketch) MemoryUsage() (sum int64) { + if c == nil { + return + } sum = int64(c.depth * c.width * 4) return } diff --git a/statistics/handle/lru_cache.go b/statistics/handle/lru_cache.go index bc1ce09c87085..ca98efc633459 100644 --- a/statistics/handle/lru_cache.go +++ b/statistics/handle/lru_cache.go @@ -387,6 +387,9 @@ func (c *innerItemLruCache) put(tblID, id int64, isIndex bool, item statistics.T c.evictIfNeeded() } }() + if itemMem.TrackingMemUsage() < 1 { + return + } isIndexSet, ok := c.elements[tblID] if !ok { c.elements[tblID] = make(map[bool]map[int64]*list.Element) @@ -432,13 +435,17 @@ func (c *innerItemLruCache) evictIfNeeded() { prev := curr.Prev() item := curr.Value.(*lruCacheItem) oldMem := item.innerMemUsage - // evict cmSketches - item.innerItem.DropEvicted() + statistics.DropEvicted(item.innerItem) newMem := item.innerItem.MemoryUsage() c.calculateCost(newMem, oldMem) - // remove from lru - c.cache.Remove(curr) - delete(c.elements[item.tblID][item.isIndex], item.id) + if newMem.TrackingMemUsage() == 0 || item.innerItem.IsAllEvicted() { + // remove from lru + c.cache.Remove(curr) + delete(c.elements[item.tblID][item.isIndex], item.id) + } else { + c.cache.PushFront(curr) + item.innerMemUsage = newMem + } if c.onEvict != nil { c.onEvict(item.tblID) } diff --git a/statistics/handle/lru_cache_test.go b/statistics/handle/lru_cache_test.go index d717b95643c23..abb6a52473716 100644 --- a/statistics/handle/lru_cache_test.go +++ b/statistics/handle/lru_cache_test.go @@ -23,14 +23,15 @@ import ( ) var ( - mockIndexMemoryUsage = int64(4) - mockColumnMemoryUsage = int64(4) - mockColumnTotalMemoryUsage = statistics.EmptyHistogramSize + 4 - mockIndexTotalMemoryUsage = statistics.EmptyHistogramSize + 4 + mockIndexCMSMemoryUsage = int64(4) + mockColumnCMSMemoryUsage = int64(4) + mockColumnTopNMemoryUsage = int64(64) + mockColumnTotalMemoryUsage = statistics.EmptyHistogramSize + mockColumnCMSMemoryUsage + mockIndexTotalMemoryUsage = statistics.EmptyHistogramSize + mockIndexCMSMemoryUsage ) // each column and index consumes 4 bytes memory -func newMockStatisticsTable(columns int, indices int) *statistics.Table { +func newMockStatisticsTable(columns int, indices int, withCMS, withTopN bool) *statistics.Table { t := &statistics.Table{} t.Columns = make(map[int64]*statistics.Column) t.Indices = make(map[int64]*statistics.Index) @@ -40,11 +41,25 @@ func newMockStatisticsTable(columns int, indices int) *statistics.Table { CMSketch: statistics.NewCMSketch(1, 1), StatsLoadedStatus: statistics.NewStatsFullLoadStatus(), } + if withCMS { + t.Columns[int64(i)].CMSketch = statistics.NewCMSketch(1, 1) + } + if withTopN { + t.Columns[int64(i)].TopN = statistics.NewTopN(1) + t.Columns[int64(i)].TopN.AppendTopN([]byte{}, 1) + } } for i := 1; i <= indices; i++ { t.Indices[int64(i)] = &statistics.Index{ - Info: &model.IndexInfo{ID: int64(i)}, - CMSketch: statistics.NewCMSketch(1, 1), + Info: &model.IndexInfo{ID: int64(i)}, + StatsLoadedStatus: statistics.NewStatsFullLoadStatus(), + } + if withCMS { + t.Indices[int64(i)].CMSketch = statistics.NewCMSketch(1, 1) + } + if withTopN { + t.Indices[int64(i)].TopN = statistics.NewTopN(1) + t.Indices[int64(i)].TopN.AppendTopN([]byte{}, 1) } } return t @@ -78,7 +93,7 @@ func TestLRUPutGetDel(t *testing.T) { capacity := int64(100) lru := newStatsLruCache(capacity) require.Equal(t, capacity, lru.capacity()) - mockTable := newMockStatisticsTable(1, 1) + mockTable := newMockStatisticsTable(1, 1, true, false) mockTableID := int64(1) lru.Put(mockTableID, mockTable) v, ok := lru.Get(mockTableID) @@ -103,10 +118,10 @@ func TestLRUPutGetDel(t *testing.T) { func TestLRUEvict(t *testing.T) { capacity := int64(24) lru := newStatsLruCache(capacity) - t1 := newMockStatisticsTable(2, 0) + t1 := newMockStatisticsTable(2, 0, true, false) require.Equal(t, t1.MemoryUsage().TotalMemUsage, 2*mockColumnTotalMemoryUsage) require.Equal(t, t1.MemoryUsage().TotalIdxTrackingMemUsage(), int64(0)) - require.Equal(t, t1.MemoryUsage().TotalColTrackingMemUsage(), 2*mockColumnMemoryUsage) + require.Equal(t, t1.MemoryUsage().TotalColTrackingMemUsage(), 2*mockColumnCMSMemoryUsage) // Put t1, assert TotalMemUsage and TotalColTrackingMemUsage lru.Put(int64(1), t1) @@ -114,21 +129,21 @@ func TestLRUEvict(t *testing.T) { require.Equal(t, lru.Cost(), t1.MemoryUsage().TotalTrackingMemUsage()) // Put t2, assert TotalMemUsage and TotalColTrackingMemUsage - t2 := newMockStatisticsTable(2, 1) + t2 := newMockStatisticsTable(2, 1, true, false) lru.Put(int64(2), t2) require.Equal(t, lru.TotalCost(), 4*mockColumnTotalMemoryUsage+1*mockIndexTotalMemoryUsage) - require.Equal(t, lru.Cost(), 4*mockColumnMemoryUsage+1*mockIndexMemoryUsage) + require.Equal(t, lru.Cost(), 4*mockColumnCMSMemoryUsage+1*mockIndexCMSMemoryUsage) // Put t3, a column of t1 should be evicted - t3 := newMockStatisticsTable(1, 1) + t3 := newMockStatisticsTable(1, 1, true, false) lru.Put(int64(3), t3) require.Equal(t, lru.Len(), 3) - require.Equal(t, t1.MemoryUsage().TotalColTrackingMemUsage(), mockColumnMemoryUsage) + require.Equal(t, t1.MemoryUsage().TotalColTrackingMemUsage(), mockColumnCMSMemoryUsage) require.Equal(t, lru.TotalCost(), t1.MemoryUsage().TotalMemUsage+t2.MemoryUsage().TotalMemUsage+t3.MemoryUsage().TotalMemUsage) - require.Equal(t, lru.Cost(), 4*mockColumnMemoryUsage+2*mockIndexMemoryUsage) + require.Equal(t, lru.Cost(), 4*mockColumnCMSMemoryUsage+2*mockIndexCMSMemoryUsage) // Put t4, all indices' cmsketch of other tables should be evicted - t4 := newMockStatisticsTable(3, 3) + t4 := newMockStatisticsTable(3, 3, true, false) lru.Put(int64(4), t4) require.Equal(t, lru.Len(), 4) require.Equal(t, t1.MemoryUsage().TotalTrackingMemUsage(), int64(0)) @@ -138,14 +153,14 @@ func TestLRUEvict(t *testing.T) { t2.MemoryUsage().TotalMemUsage+ t3.MemoryUsage().TotalMemUsage+ t4.MemoryUsage().TotalMemUsage) - require.Equal(t, lru.Cost(), 3*mockColumnMemoryUsage+3*mockIndexMemoryUsage) + require.Equal(t, lru.Cost(), 3*mockColumnCMSMemoryUsage+3*mockIndexCMSMemoryUsage) } func TestLRUCopy(t *testing.T) { lru := newStatsLruCache(1000) tables := make([]*statistics.Table, 0) for i := 0; i < 5; i++ { - tables = append(tables, newMockStatisticsTable(1, 1)) + tables = append(tables, newMockStatisticsTable(1, 1, true, false)) } // insert 1,2,3 into old lru @@ -181,42 +196,42 @@ func TestLRUCopy(t *testing.T) { func TestLRUFreshMemUsage(t *testing.T) { lru := newStatsLruCache(1000) - t1 := newMockStatisticsTable(1, 1) - t2 := newMockStatisticsTable(2, 2) - t3 := newMockStatisticsTable(3, 3) + t1 := newMockStatisticsTable(1, 1, true, false) + t2 := newMockStatisticsTable(2, 2, true, false) + t3 := newMockStatisticsTable(3, 3, true, false) lru.Put(int64(1), t1) lru.Put(int64(2), t2) lru.Put(int64(3), t3) require.Equal(t, lru.TotalCost(), 6*mockColumnTotalMemoryUsage+6*mockIndexTotalMemoryUsage) - require.Equal(t, lru.Cost(), 6*mockIndexMemoryUsage+6*mockColumnMemoryUsage) + require.Equal(t, lru.Cost(), 6*mockIndexCMSMemoryUsage+6*mockColumnCMSMemoryUsage) mockTableAppendColumn(t1) lru.FreshMemUsage() require.Equal(t, lru.TotalCost(), 7*mockColumnTotalMemoryUsage+6*mockIndexTotalMemoryUsage) - require.Equal(t, lru.Cost(), 6*mockIndexMemoryUsage+7*mockColumnMemoryUsage) + require.Equal(t, lru.Cost(), 6*mockIndexCMSMemoryUsage+7*mockColumnCMSMemoryUsage) mockTableAppendIndex(t1) lru.FreshMemUsage() require.Equal(t, lru.TotalCost(), 7*mockColumnTotalMemoryUsage+7*mockIndexTotalMemoryUsage) - require.Equal(t, lru.Cost(), 7*mockIndexMemoryUsage+7*mockColumnMemoryUsage) + require.Equal(t, lru.Cost(), 7*mockIndexCMSMemoryUsage+7*mockColumnCMSMemoryUsage) mockTableRemoveColumn(t1) lru.Put(int64(1), t1) require.Equal(t, lru.TotalCost(), 6*mockColumnTotalMemoryUsage+7*mockIndexTotalMemoryUsage) - require.Equal(t, lru.Cost(), 7*mockIndexMemoryUsage+6*mockColumnMemoryUsage) + require.Equal(t, lru.Cost(), 7*mockIndexCMSMemoryUsage+6*mockColumnCMSMemoryUsage) mockTableRemoveIndex(t1) lru.Put(int64(1), t1) require.Equal(t, lru.TotalCost(), 6*mockColumnTotalMemoryUsage+6*mockIndexTotalMemoryUsage) - require.Equal(t, lru.Cost(), 6*mockIndexMemoryUsage+6*mockColumnMemoryUsage) + require.Equal(t, lru.Cost(), 6*mockIndexCMSMemoryUsage+6*mockColumnCMSMemoryUsage) } func TestLRUPutTooBig(t *testing.T) { lru := newStatsLruCache(1) - mockTable := newMockStatisticsTable(1, 1) + mockTable := newMockStatisticsTable(1, 1, true, false) // put mockTable, the index should be evicted lru.Put(int64(1), mockTable) _, ok := lru.Get(int64(1)) require.True(t, ok) - require.Equal(t, lru.TotalCost(), mockColumnTotalMemoryUsage-mockColumnMemoryUsage+mockIndexTotalMemoryUsage-mockIndexMemoryUsage) + require.Equal(t, lru.TotalCost(), mockColumnTotalMemoryUsage-mockColumnCMSMemoryUsage+mockIndexTotalMemoryUsage-mockIndexCMSMemoryUsage) require.Equal(t, lru.Cost(), int64(0)) require.Equal(t, mockTable.MemoryUsage().TotalTrackingMemUsage(), int64(0)) } @@ -224,9 +239,9 @@ func TestLRUPutTooBig(t *testing.T) { func TestCacheLen(t *testing.T) { capacity := int64(12) stats := newStatsLruCache(capacity) - t1 := newMockStatisticsTable(2, 1) + t1 := newMockStatisticsTable(2, 1, true, false) stats.Put(int64(1), t1) - t2 := newMockStatisticsTable(1, 1) + t2 := newMockStatisticsTable(1, 1, true, false) // put t2, t1 should be evicted 2 items and still exists in the list stats.Put(int64(2), t2) require.Equal(t, stats.lru.cache.Len(), 3) @@ -234,7 +249,7 @@ func TestCacheLen(t *testing.T) { require.Equal(t, stats.Len(), 2) // put t3, t1/t2 should be evicted all items and disappeared from the list - t3 := newMockStatisticsTable(2, 1) + t3 := newMockStatisticsTable(2, 1, true, false) stats.Put(int64(3), t3) require.Equal(t, stats.lru.cache.Len(), 3) require.Equal(t, t1.MemoryUsage().TotalTrackingMemUsage(), int64(0)) @@ -245,9 +260,9 @@ func TestCacheLen(t *testing.T) { func TestLRUMove(t *testing.T) { capacity := int64(100) s := newStatsLruCache(capacity) - t1 := newMockStatisticsTable(1, 1) + t1 := newMockStatisticsTable(1, 1, true, false) t1ID := int64(1) - t2 := newMockStatisticsTable(1, 1) + t2 := newMockStatisticsTable(1, 1, true, false) t2ID := int64(2) s.Put(t1ID, t1) s.Put(t2ID, t2) @@ -259,3 +274,44 @@ func TestLRUMove(t *testing.T) { front = s.lru.cache.Front().Value.(*lruCacheItem) require.Equal(t, t1ID, front.tblID) } + +func TestLRUEvictPolicy(t *testing.T) { + capacity := int64(999) + s := newStatsLruCache(capacity) + t1 := newMockStatisticsTable(1, 0, true, true) + s.Put(1, t1) + require.Equal(t, s.TotalCost(), mockColumnTotalMemoryUsage+mockColumnTopNMemoryUsage) + require.Equal(t, s.Cost(), mockColumnCMSMemoryUsage+mockColumnTopNMemoryUsage) + cost := s.Cost() + // assert column's cms got evicted and topn remained + s.SetCapacity(cost - 1) + require.Equal(t, s.Cost(), mockColumnTopNMemoryUsage) + require.Nil(t, t1.Columns[1].CMSketch) + require.True(t, t1.Columns[1].IsCMSEvicted()) + require.NotNil(t, t1.Columns[1].TopN) + require.False(t, t1.Columns[1].IsTopNEvicted()) + // assert both column's cms and topn got evicted + s.SetCapacity(1) + require.Equal(t, s.Cost(), int64(0)) + require.Nil(t, t1.Columns[1].CMSketch) + require.True(t, t1.Columns[1].IsCMSEvicted()) + require.Nil(t, t1.Columns[1].TopN) + require.True(t, t1.Columns[1].IsTopNEvicted()) + + s = newStatsLruCache(capacity) + t2 := newMockStatisticsTable(0, 1, true, true) + s.Put(2, t2) + require.Equal(t, s.TotalCost(), mockIndexTotalMemoryUsage+mockColumnTopNMemoryUsage) + require.Equal(t, s.Cost(), mockIndexCMSMemoryUsage+mockColumnTopNMemoryUsage) + cost = s.Cost() + // assert index's cms got evicted and topn remained + s.SetCapacity(cost - 1) + require.Equal(t, s.Cost(), mockColumnTopNMemoryUsage) + require.Nil(t, t2.Indices[1].CMSketch) + require.NotNil(t, t2.Indices[1].TopN) + // assert both column's cms and topn got evicted + s.SetCapacity(1) + require.Equal(t, s.Cost(), int64(0)) + require.Nil(t, t2.Indices[1].CMSketch) + require.Nil(t, t2.Indices[1].TopN) +} diff --git a/statistics/histogram.go b/statistics/histogram.go index 01f970b34854e..46c83a8397cb4 100644 --- a/statistics/histogram.go +++ b/statistics/histogram.go @@ -1106,6 +1106,11 @@ func (c *Column) MemoryUsage() CacheItemMemoryUsage { columnMemUsage.CMSketchMemUsage = cmSketchMemUsage sum += cmSketchMemUsage } + if c.TopN != nil { + topnMemUsage := c.TopN.MemoryUsage() + columnMemUsage.TopNMemUsage = topnMemUsage + sum += topnMemUsage + } if c.FMSketch != nil { fmSketchMemUsage := c.FMSketch.MemoryUsage() columnMemUsage.FMSketchMemUsage = fmSketchMemUsage @@ -1316,12 +1321,63 @@ func (c *Column) ItemID() int64 { // DropEvicted implements TableCacheItem // DropEvicted drops evicted structures func (c *Column) DropEvicted() { - if c.StatsVer < Version2 && c.IsStatsInitialized() { - c.CMSketch = nil - c.evictedStatus = onlyCmsEvicted + if !c.statsInitialized { + return + } + switch c.evictedStatus { + case allLoaded: + if c.CMSketch != nil && c.StatsVer < Version2 { + c.dropCMS() + return + } + // For stats version2, there is no cms thus we directly drop topn + c.dropTopN() + return + case onlyCmsEvicted: + c.dropTopN() + return + default: + return + } +} + +func (c *Column) dropCMS() { + c.CMSketch = nil + c.evictedStatus = onlyCmsEvicted +} + +func (c *Column) dropTopN() { + originTopNNum := int64(c.TopN.Num()) + c.TopN = nil + if len(c.Histogram.Buckets) == 0 && originTopNNum >= c.Histogram.NDV { + // This indicates column has topn instead of histogram + c.evictedStatus = allEvicted + } else { + c.evictedStatus = onlyHistRemained } } +// IsAllEvicted indicates whether all stats evicted +func (c *Column) IsAllEvicted() bool { + return c.statsInitialized && c.evictedStatus >= allEvicted +} + +func (c *Column) getEvictedStatus() int { + return c.evictedStatus +} + +func (c *Column) isStatsInitialized() bool { + return c.statsInitialized +} + +func (c *Column) statsVer() int64 { + return c.StatsVer +} + +func (c *Column) isCMSExist() bool { + return c.CMSketch != nil +} + // Index represents an index histogram. type Index struct { Histogram @@ -1342,20 +1398,46 @@ func (idx *Index) ItemID() int64 { return idx.Info.ID } -// DropEvicted implements TableCacheItem -// DropEvicted drops evicted structures -func (idx *Index) DropEvicted() { +// IsAllEvicted indicates whether all stats evicted +func (idx *Index) IsAllEvicted() bool { + return idx.statsInitialized && idx.evictedStatus >= allEvicted +} + +func (idx *Index) dropCMS() { idx.CMSketch = nil + idx.evictedStatus = onlyCmsEvicted +} + +func (idx *Index) dropTopN() { + originTopNNum := int64(idx.TopN.Num()) + idx.TopN = nil + if len(idx.Histogram.Buckets) == 0 && originTopNNum >= idx.Histogram.NDV { + // This indicates index has topn instead of histogram + idx.evictedStatus = allEvicted + } else { + idx.evictedStatus = onlyHistRemained + } +} + +func (idx *Index) getEvictedStatus() int { + return idx.evictedStatus +} + +func (idx *Index) isStatsInitialized() bool { + return idx.statsInitialized +} + +func (idx *Index) statsVer() int64 { + return idx.StatsVer +} + +func (idx *Index) isCMSExist() bool { + return idx.CMSketch != nil } // IsEvicted returns whether index statistics got evicted func (idx *Index) IsEvicted() bool { - switch idx.StatsVer { - case Version1: - return idx.CMSketch == nil - default: - return false - } + return idx.evictedStatus != allLoaded } func (idx *Index) String() string { @@ -1403,6 +1485,11 @@ func (idx *Index) MemoryUsage() CacheItemMemoryUsage { indexMemUsage.CMSketchMemUsage = cmSketchMemUsage sum += cmSketchMemUsage } + if idx.TopN != nil { + topnMemUsage := idx.TopN.MemoryUsage() + indexMemUsage.TopNMemUsage = topnMemUsage + sum += topnMemUsage + } indexMemUsage.TotalMemUsage = sum return indexMemUsage } @@ -2303,7 +2390,7 @@ func MergePartitionHist2GlobalHist(sc *stmtctx.StatementContext, hists []*Histog const ( allLoaded = iota onlyCmsEvicted - //onlyHistRemained + onlyHistRemained allEvicted ) @@ -2347,6 +2434,11 @@ func (s StatsLoadedStatus) IsCMSEvicted() bool { return s.statsInitialized && s.evictedStatus >= onlyCmsEvicted } +// IsTopNEvicted indicates whether the topn got evicted now. +func (s StatsLoadedStatus) IsTopNEvicted() bool { + return s.statsInitialized && s.evictedStatus >= onlyHistRemained +} + // IsFullLoad indicates whether the stats are full loaded func (s StatsLoadedStatus) IsFullLoad() bool { return s.statsInitialized && s.evictedStatus == allLoaded diff --git a/statistics/table.go b/statistics/table.go index 1adcdb01dcb96..ac3ffa8442b28 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -144,8 +144,37 @@ func (t *TableMemoryUsage) TotalTrackingMemUsage() int64 { // TableCacheItem indicates the unit item stored in statsCache, eg: Column/Index type TableCacheItem interface { ItemID() int64 - DropEvicted() MemoryUsage() CacheItemMemoryUsage + IsAllEvicted() bool + + dropCMS() + dropTopN() + isStatsInitialized() bool + getEvictedStatus() int + statsVer() int64 + isCMSExist() bool +} + +// DropEvicted drop stats for table column/index +func DropEvicted(item TableCacheItem) { + if !item.isStatsInitialized() { + return + } + switch item.getEvictedStatus() { + case allLoaded: + if item.isCMSExist() && item.statsVer() < Version2 { + item.dropCMS() + return + } + // For stats version2, there is no cms thus we directly drop topn + item.dropTopN() + return + case onlyCmsEvicted: + item.dropTopN() + return + default: + return + } } // CacheItemMemoryUsage indicates the memory usage of TableCacheItem @@ -161,6 +190,7 @@ type ColumnMemUsage struct { HistogramMemUsage int64 CMSketchMemUsage int64 FMSketchMemUsage int64 + TopNMemUsage int64 TotalMemUsage int64 } @@ -176,7 +206,7 @@ func (c *ColumnMemUsage) ItemID() int64 { // TrackingMemUsage implements CacheItemMemoryUsage func (c *ColumnMemUsage) TrackingMemUsage() int64 { - return c.CMSketchMemUsage + return c.CMSketchMemUsage + c.TopNMemUsage } // IndexMemUsage records index memory usage @@ -184,6 +214,7 @@ type IndexMemUsage struct { IndexID int64 HistogramMemUsage int64 CMSketchMemUsage int64 + TopNMemUsage int64 TotalMemUsage int64 } @@ -199,7 +230,7 @@ func (c *IndexMemUsage) ItemID() int64 { // TrackingMemUsage implements CacheItemMemoryUsage func (c *IndexMemUsage) TrackingMemUsage() int64 { - return c.CMSketchMemUsage + return c.CMSketchMemUsage + c.TopNMemUsage } // MemoryUsage returns the total memory usage of this Table. From bdc6397023e817de3d1400a9ab430d742ec5a790 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Wed, 13 Jul 2022 12:33:06 +0800 Subject: [PATCH 04/27] executor: support tidb memory debug mode (#35322) ref pingcap/tidb#33877 --- executor/explain.go | 152 +++++++++++++++++++++++++++++ planner/core/common_plans.go | 7 ++ sessionctx/variable/session.go | 6 ++ sessionctx/variable/sysvar.go | 8 ++ sessionctx/variable/sysvar_test.go | 10 ++ sessionctx/variable/tidb_vars.go | 10 ++ util/memory/tracker.go | 14 +++ 7 files changed, 207 insertions(+) diff --git a/executor/explain.go b/executor/explain.go index 57d4261cf6745..1fef25865bc59 100644 --- a/executor/explain.go +++ b/executor/explain.go @@ -16,11 +16,22 @@ package executor import ( "context" + "os" + "path/filepath" + "runtime" + rpprof "runtime/pprof" + "strconv" + "sync" + "time" "github.com/pingcap/errors" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/mathutil" + "github.com/pingcap/tidb/util/memory" + "go.uber.org/zap" ) // ExplainExec represents an explain executor. @@ -89,6 +100,24 @@ func (e *ExplainExec) executeAnalyzeExec(ctx context.Context) (err error) { } } }() + if minHeapInUse, alarmRatio := e.ctx.GetSessionVars().MemoryDebugModeMinHeapInUse, e.ctx.GetSessionVars().MemoryDebugModeAlarmRatio; minHeapInUse != 0 && alarmRatio != 0 { + memoryDebugModeCtx, cancel := context.WithCancel(ctx) + waitGroup := sync.WaitGroup{} + waitGroup.Add(1) + defer func() { + // Notify and wait debug goroutine exit. + cancel() + waitGroup.Wait() + }() + go (&memoryDebugModeHandler{ + ctx: memoryDebugModeCtx, + minHeapInUse: mathutil.Abs(minHeapInUse), + alarmRatio: alarmRatio, + autoGC: minHeapInUse > 0, + memTracker: e.ctx.GetSessionVars().StmtCtx.MemTracker, + wg: &waitGroup, + }).run() + } e.executed = true chk := newFirstChunk(e.analyzeExec) for { @@ -123,3 +152,126 @@ func (e *ExplainExec) getAnalyzeExecToExecutedNoDelay() Executor { } return nil } + +type memoryDebugModeHandler struct { + ctx context.Context + minHeapInUse int64 + alarmRatio int64 + autoGC bool + wg *sync.WaitGroup + memTracker *memory.Tracker + + infoField []zap.Field +} + +func (h *memoryDebugModeHandler) fetchCurrentMemoryUsage(gc bool) (heapInUse, trackedMem uint64) { + if gc { + runtime.GC() + } + instanceStats := &runtime.MemStats{} + runtime.ReadMemStats(instanceStats) + heapInUse = instanceStats.HeapInuse + trackedMem = uint64(h.memTracker.BytesConsumed()) + return +} + +func (h *memoryDebugModeHandler) genInfo(status string, needProfile bool, heapInUse, trackedMem int64) (fields []zap.Field, err error) { + var fileName string + h.infoField = h.infoField[:0] + h.infoField = append(h.infoField, zap.String("sql", status)) + h.infoField = append(h.infoField, zap.String("heap in use", memory.FormatBytes(heapInUse))) + h.infoField = append(h.infoField, zap.String("tracked memory", memory.FormatBytes(trackedMem))) + if needProfile { + fileName, err = getHeapProfile() + h.infoField = append(h.infoField, zap.String("heap profile", fileName)) + } + return h.infoField, err +} + +func (h *memoryDebugModeHandler) run() { + var err error + var fields []zap.Field + defer func() { + heapInUse, trackedMem := h.fetchCurrentMemoryUsage(true) + if err == nil { + fields, err := h.genInfo("finished", true, int64(heapInUse), int64(trackedMem)) + logutil.BgLogger().Info("Memory Debug Mode", fields...) + if err != nil { + logutil.BgLogger().Error("Memory Debug Mode Exit", zap.Error(err)) + } + } else { + fields, err := h.genInfo("debug_mode_error", false, int64(heapInUse), int64(trackedMem)) + logutil.BgLogger().Error("Memory Debug Mode", fields...) + logutil.BgLogger().Error("Memory Debug Mode Exit", zap.Error(err)) + } + h.wg.Done() + }() + + logutil.BgLogger().Info("Memory Debug Mode", + zap.String("sql", "started"), + zap.Bool("autoGC", h.autoGC), + zap.String("minHeapInUse", memory.FormatBytes(h.minHeapInUse)), + zap.Int64("alarmRatio", h.alarmRatio), + ) + ticker, loop := time.NewTicker(5*time.Second), 0 + for { + select { + case <-h.ctx.Done(): + return + case <-ticker.C: + heapInUse, trackedMem := h.fetchCurrentMemoryUsage(h.autoGC) + loop++ + if loop%6 == 0 { + fields, err = h.genInfo("running", false, int64(heapInUse), int64(trackedMem)) + logutil.BgLogger().Info("Memory Debug Mode", fields...) + if err != nil { + return + } + } + + if !h.autoGC { + if heapInUse > uint64(h.minHeapInUse) && trackedMem/100*uint64(100+h.alarmRatio) < heapInUse { + fields, err = h.genInfo("warning", true, int64(heapInUse), int64(trackedMem)) + logutil.BgLogger().Warn("Memory Debug Mode", fields...) + if err != nil { + return + } + } + } else { + if heapInUse > uint64(h.minHeapInUse) && trackedMem/100*uint64(100+h.alarmRatio) < heapInUse { + fields, err = h.genInfo("warning", true, int64(heapInUse), int64(trackedMem)) + logutil.BgLogger().Warn("Memory Debug Mode", fields...) + if err != nil { + return + } + ts := h.memTracker.SearchTrackerConsumedMoreThanNBytes(h.minHeapInUse / 5) + logs := make([]zap.Field, 0, len(ts)) + for _, t := range ts { + logs = append(logs, zap.String("Executor_"+strconv.Itoa(t.Label()), memory.FormatBytes(t.BytesConsumed()))) + } + logutil.BgLogger().Warn("Memory Debug Mode, Log all trackers that consumes more than threshold * 20%", logs...) + } + } + } + } +} + +func getHeapProfile() (fileName string, err error) { + tempDir := filepath.Join(config.GetGlobalConfig().TempStoragePath, "record") + timeString := time.Now().Format(time.RFC3339) + fileName = filepath.Join(tempDir, "heapGC"+timeString) + f, err := os.Create(fileName) + if err != nil { + return "", err + } + p := rpprof.Lookup("heap") + err = p.WriteTo(f, 0) + if err != nil { + return "", err + } + err = f.Close() + if err != nil { + return "", err + } + return fileName, nil +} diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 89873ff086650..e5c04aa6f538b 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -1238,6 +1238,13 @@ func (e *Explain) RenderResult() error { if e.Rows == nil || e.Analyze { flat := FlattenPhysicalPlan(e.TargetPlan, true) e.explainFlatPlanInRowFormat(flat) + if e.Analyze && + e.SCtx().GetSessionVars().MemoryDebugModeMinHeapInUse != 0 && + e.SCtx().GetSessionVars().MemoryDebugModeAlarmRatio > 0 { + row := e.Rows[0] + tracker := e.SCtx().GetSessionVars().StmtCtx.MemTracker + row[7] = row[7] + "(Total: " + tracker.FormatBytes(tracker.MaxConsumed()) + ")" + } } case types.ExplainFormatDOT: if physicalPlan, ok := e.TargetPlan.(PhysicalPlan); ok { diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 04e1279e37410..add261a90d449 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1165,6 +1165,12 @@ type SessionVars struct { // RequestSourceType is the type of inner request. RequestSourceType string + + // MemoryDebugModeMinHeapInUse indicated the minimum heapInUse threshold that triggers the memoryDebugMode. + MemoryDebugModeMinHeapInUse int64 + // MemoryDebugModeAlarmRatio indicated the allowable bias ratio of memory tracking accuracy check. + // When `(memory trakced by tidb) * (1+MemoryDebugModeAlarmRatio) < actual heapInUse`, an alarm log will be recorded. + MemoryDebugModeAlarmRatio int64 } // InitStatementContext initializes a StatementContext, the object is reused to reduce allocation. diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 98c217cf3cf5a..dba7781dfbe1f 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1667,6 +1667,14 @@ var defaultSysVars = []*SysVar{ metrics.ToggleSimplifiedMode(TiDBOptOn(s)) return nil }}, + {Scope: ScopeSession, Name: TiDBMemoryDebugModeMinHeapInUse, Value: strconv.Itoa(0), Type: TypeInt, MinValue: math.MinInt64, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + s.MemoryDebugModeMinHeapInUse = TidbOptInt64(val, 0) + return nil + }}, + {Scope: ScopeSession, Name: TiDBMemoryDebugModeAlarmRatio, Value: strconv.Itoa(0), Type: TypeInt, MinValue: 0, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + s.MemoryDebugModeAlarmRatio = TidbOptInt64(val, 0) + return nil + }}, } // FeedbackProbability points to the FeedbackProbability in statistics package. diff --git a/sessionctx/variable/sysvar_test.go b/sessionctx/variable/sysvar_test.go index 1146732e6030c..ebf0fd2587624 100644 --- a/sessionctx/variable/sysvar_test.go +++ b/sessionctx/variable/sysvar_test.go @@ -1059,3 +1059,13 @@ func TestTiDBCommitterConcurrency(t *testing.T) { require.Equal(t, val, fmt.Sprintf("%d", expected)) require.NoError(t, err) } + +func TestDefaultMemoryDebugModeValue(t *testing.T) { + vars := NewSessionVars() + val, err := GetSessionOrGlobalSystemVar(vars, TiDBMemoryDebugModeMinHeapInUse) + require.NoError(t, err) + require.Equal(t, val, "0") + val, err = GetSessionOrGlobalSystemVar(vars, TiDBMemoryDebugModeAlarmRatio) + require.NoError(t, err) + require.Equal(t, val, "0") +} diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 2e55dfdb2353d..54b49f840f494 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -684,6 +684,16 @@ const ( // TiDBSimplifiedMetrics controls whether to unregister some unused metrics. TiDBSimplifiedMetrics = "tidb_simplified_metrics" + + // TiDBMemoryDebugModeMinHeapInUse is used to set tidb memory debug mode trigger threshold. + // When set to 0, the function is disabled. + // When set to a negative integer, use memory debug mode to detect the issue of frequent allocation and release of memory. + // We do not actively trigger gc, and check whether the `tracker memory * (1+bias ratio) > heap in use` each 5s. + // When set to a positive integer, use memory debug mode to detect the issue of memory tracking inaccurate. + // We trigger runtime.GC() each 5s, and check whether the `tracker memory * (1+bias ratio) > heap in use`. + TiDBMemoryDebugModeMinHeapInUse = "tidb_memory_debug_mode_min_heap_inuse" + // TiDBMemoryDebugModeAlarmRatio is used set tidb memory debug mode bias ratio. Treat memory bias less than this ratio as noise. + TiDBMemoryDebugModeAlarmRatio = "tidb_memory_debug_mode_alarm_ratio" ) // TiDB vars that have only global scope diff --git a/util/memory/tracker.go b/util/memory/tracker.go index 75dba11cea11f..d0b1e17c22bc9 100644 --- a/util/memory/tracker.go +++ b/util/memory/tracker.go @@ -418,6 +418,20 @@ func (t *Tracker) SearchTrackerWithoutLock(label int) *Tracker { return nil } +// SearchTrackerConsumedMoreThanNBytes searches the specific tracker that consumes more than NBytes. +func (t *Tracker) SearchTrackerConsumedMoreThanNBytes(limit int64) (res []*Tracker) { + t.mu.Lock() + defer t.mu.Unlock() + for _, childSlice := range t.mu.children { + for _, tracker := range childSlice { + if tracker.BytesConsumed() > limit { + res = append(res, tracker) + } + } + } + return +} + // String returns the string representation of this Tracker tree. func (t *Tracker) String() string { buffer := bytes.NewBufferString("\n") From 7c45e671af91a3e088c45c118f17f3368a687e68 Mon Sep 17 00:00:00 2001 From: lizhenhuan <1916038084@qq.com> Date: Wed, 13 Jul 2022 12:53:05 +0800 Subject: [PATCH 05/27] expression: pushdown reverse to TiFlash (#35738) close pingcap/tidb#35754 --- expression/expr_to_pb_test.go | 10 +++++ expression/expression.go | 6 ++- planner/core/integration_test.go | 70 ++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index c69557b1bbb46..c65e43b408390 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -1136,6 +1136,16 @@ func TestExprPushDownToFlash(t *testing.T) { require.NoError(t, err) exprs = append(exprs, function) + // ReverseUTF8 test + function, err = NewFunction(mock.NewContext(), ast.Reverse, types.NewFieldType(mysql.TypeString), stringColumn) + require.NoError(t, err) + exprs = append(exprs, function) + + // Reverse + function, err = NewFunction(mock.NewContext(), ast.Reverse, types.NewFieldType(mysql.TypeBlob), stringColumn) + require.NoError(t, err) + exprs = append(exprs, function) + pushed, remained = PushDownExprs(sc, exprs, client, kv.TiFlash) require.Len(t, pushed, len(exprs)) require.Len(t, remained, 0) diff --git a/expression/expression.go b/expression/expression.go index ba4384f981096..1a5c3248c889a 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -1070,7 +1070,7 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool { return false } return true - case ast.Substr, ast.Substring, ast.Left, ast.Right, ast.CharLength, ast.SubstringIndex: + case ast.Substr, ast.Substring, ast.Left, ast.Right, ast.CharLength, ast.SubstringIndex, ast.Reverse: switch function.Function.PbCode() { case tipb.ScalarFuncSig_LeftUTF8, @@ -1078,7 +1078,9 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool { tipb.ScalarFuncSig_CharLengthUTF8, tipb.ScalarFuncSig_Substring2ArgsUTF8, tipb.ScalarFuncSig_Substring3ArgsUTF8, - tipb.ScalarFuncSig_SubstringIndex: + tipb.ScalarFuncSig_SubstringIndex, + tipb.ScalarFuncSig_ReverseUTF8, + tipb.ScalarFuncSig_Reverse: return true } case ast.Cast: diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index d43ed649f6e6b..e059d97711560 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -3212,6 +3212,76 @@ func TestDistinctScalarFunctionPushDown(t *testing.T) { )) } +func TestReverseUTF8PushDownToTiFlash(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a varchar(256))") + tk.MustExec("insert into t values('pingcap')") + tk.MustExec("set @@tidb_allow_mpp=1; set @@tidb_enforce_mpp=1;") + tk.MustExec("set @@tidb_isolation_read_engines = 'tiflash'") + + // Create virtual tiflash replica info. + dom := domain.GetDomain(tk.Session()) + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + require.True(t, exists) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "t" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + rows := [][]interface{}{ + {"TableReader_9", "root", "data:ExchangeSender_8"}, + {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, + {" └─Projection_4", "mpp[tiflash]", "reverse(test.t.a)->Column#3"}, + {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, + } + + tk.MustQuery("explain select reverse(a) from t;").CheckAt([]int{0, 2, 4}, rows) +} + +func TestReversePushDownToTiFlash(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a binary(32))") + tk.MustExec("insert into t values('pingcap')") + tk.MustExec("set @@tidb_allow_mpp=1; set @@tidb_enforce_mpp=1;") + tk.MustExec("set @@tidb_isolation_read_engines = 'tiflash'") + + // Create virtual tiflash replica info. + dom := domain.GetDomain(tk.Session()) + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + require.True(t, exists) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "t" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + rows := [][]interface{}{ + {"TableReader_9", "root", "data:ExchangeSender_8"}, + {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, + {" └─Projection_4", "mpp[tiflash]", "reverse(test.t.a)->Column#3"}, + {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, + } + + tk.MustQuery("explain select reverse(a) from t;").CheckAt([]int{0, 2, 4}, rows) +} + func TestExplainAnalyzePointGet(t *testing.T) { store, clean := testkit.CreateMockStore(t) defer clean() From ffc17f745be1d9e402c9dbed60c2391c6db96a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=B6=85?= Date: Wed, 13 Jul 2022 13:13:05 +0800 Subject: [PATCH 06/27] txn: move some methods in `session_txn` to internal pacakge (#36034) close pingcap/tidb#36033 --- sessiontxn/future.go | 23 ++++++ sessiontxn/{ => internal}/txn.go | 70 +++------------- sessiontxn/isolation/base.go | 54 +++++++++++-- sessiontxn/isolation/readcommitted.go | 6 +- sessiontxn/isolation/repeatable_read.go | 2 +- sessiontxn/staleread/provider.go | 7 +- sessiontxn/txn_manager_test.go | 79 +++++++++++++++++-- .../sessiontest/temporary_table_test.go | 64 --------------- 8 files changed, 160 insertions(+), 145 deletions(-) create mode 100644 sessiontxn/future.go rename sessiontxn/{ => internal}/txn.go (57%) diff --git a/sessiontxn/future.go b/sessiontxn/future.go new file mode 100644 index 0000000000000..d29d3e5614f0f --- /dev/null +++ b/sessiontxn/future.go @@ -0,0 +1,23 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sessiontxn + +// ConstantFuture implements oracle.Future +type ConstantFuture uint64 + +// Wait returns a constant ts +func (n ConstantFuture) Wait() (uint64, error) { + return uint64(n), nil +} diff --git a/sessiontxn/txn.go b/sessiontxn/internal/txn.go similarity index 57% rename from sessiontxn/txn.go rename to sessiontxn/internal/txn.go index 00ca355228ca9..00db4561b979b 100644 --- a/sessiontxn/txn.go +++ b/sessiontxn/internal/txn.go @@ -12,66 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sessiontxn +package internal import ( "context" - "github.com/opentracing/opentracing-go" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/table/temptable" "github.com/pingcap/tidb/util/logutil" - "github.com/tikv/client-go/v2/oracle" "go.uber.org/zap" ) -// ConstantFuture implements oracle.Future -type ConstantFuture uint64 - -// Wait returns a constant ts -func (n ConstantFuture) Wait() (uint64, error) { - return uint64(n), nil -} - -// FuncFuture implements oracle.Future -type FuncFuture func() (uint64, error) - -// Wait returns a ts got from the func -func (f FuncFuture) Wait() (uint64, error) { - return f() -} - -// NewOracleFuture creates new future according to the scope and the session context -func NewOracleFuture(ctx context.Context, sctx sessionctx.Context, scope string) oracle.Future { - if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { - span1 := span.Tracer().StartSpan("sessiontxn.NewOracleFuture", opentracing.ChildOf(span.Context())) - defer span1.Finish() - ctx = opentracing.ContextWithSpan(ctx, span1) - } - - oracleStore := sctx.GetStore().GetOracle() - option := &oracle.Option{TxnScope: scope} - - if sctx.GetSessionVars().LowResolutionTSO { - return oracleStore.GetLowResolutionTimestampAsync(ctx, option) +// SetTxnAssertionLevel sets assertion level of a transactin. Note that assertion level should be set only once just +// after creating a new transaction. +func SetTxnAssertionLevel(txn kv.Transaction, assertionLevel variable.AssertionLevel) { + switch assertionLevel { + case variable.AssertionLevelOff: + txn.SetOption(kv.AssertionLevel, kvrpcpb.AssertionLevel_Off) + case variable.AssertionLevelFast: + txn.SetOption(kv.AssertionLevel, kvrpcpb.AssertionLevel_Fast) + case variable.AssertionLevelStrict: + txn.SetOption(kv.AssertionLevel, kvrpcpb.AssertionLevel_Strict) } - return oracleStore.GetTimestampAsync(ctx, option) -} - -// CanReuseTxnWhenExplicitBegin returns whether we should reuse the txn when starting a transaction explicitly -func CanReuseTxnWhenExplicitBegin(sctx sessionctx.Context) bool { - sessVars := sctx.GetSessionVars() - txnCtx := sessVars.TxnCtx - // If BEGIN is the first statement in TxnCtx, we can reuse the existing transaction, without the - // need to call NewTxn, which commits the existing transaction and begins a new one. - // If the last un-committed/un-rollback transaction is a time-bounded read-only transaction, we should - // always create a new transaction. - // If the variable `tidb_snapshot` is set, we should always create a new transaction because the current txn may be - // initialized with snapshot ts. - return txnCtx.History == nil && !txnCtx.IsStaleness && sessVars.SnapshotTS == 0 } // CommitBeforeEnterNewTxn is called before entering a new transaction. It checks whether the old @@ -108,16 +73,3 @@ func GetSnapshotWithTS(s sessionctx.Context, ts uint64) kv.Snapshot { } return snap } - -// SetTxnAssertionLevel sets assertion level of a transactin. Note that assertion level should be set only once just -// after creating a new transaction. -func SetTxnAssertionLevel(txn kv.Transaction, assertionLevel variable.AssertionLevel) { - switch assertionLevel { - case variable.AssertionLevelOff: - txn.SetOption(kv.AssertionLevel, kvrpcpb.AssertionLevel_Off) - case variable.AssertionLevelFast: - txn.SetOption(kv.AssertionLevel, kvrpcpb.AssertionLevel_Fast) - case variable.AssertionLevelStrict: - txn.SetOption(kv.AssertionLevel, kvrpcpb.AssertionLevel_Strict) - } -} diff --git a/sessiontxn/isolation/base.go b/sessiontxn/isolation/base.go index e8f18bf75059c..572bc218f754b 100644 --- a/sessiontxn/isolation/base.go +++ b/sessiontxn/isolation/base.go @@ -18,6 +18,7 @@ import ( "context" "time" + "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/infoschema" @@ -26,6 +27,7 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/sessiontxn" + "github.com/pingcap/tidb/sessiontxn/internal" "github.com/pingcap/tidb/sessiontxn/staleread" "github.com/pingcap/tidb/table/temptable" "github.com/tikv/client-go/v2/oracle" @@ -72,19 +74,19 @@ func (p *baseTxnContextProvider) OnInitialize(ctx context.Context, tp sessiontxn // There are two main steps here to enter a new txn: // 1. prepareTxnWithOracleTS // 2. ActivateTxn - if err := sessiontxn.CommitBeforeEnterNewTxn(p.ctx, p.sctx); err != nil { + if err := internal.CommitBeforeEnterNewTxn(p.ctx, p.sctx); err != nil { return err } if err := p.prepareTxnWithOracleTS(); err != nil { return err } case sessiontxn.EnterNewTxnWithBeginStmt: - if !sessiontxn.CanReuseTxnWhenExplicitBegin(p.sctx) { + if !canReuseTxnWhenExplicitBegin(p.sctx) { // As we will enter a new txn, we need to commit the old txn if it's still valid. // There are two main steps here to enter a new txn: // 1. prepareTxnWithOracleTS // 2. ActivateTxn - if err := sessiontxn.CommitBeforeEnterNewTxn(p.ctx, p.sctx); err != nil { + if err := internal.CommitBeforeEnterNewTxn(p.ctx, p.sctx); err != nil { return err } if err := p.prepareTxnWithOracleTS(); err != nil { @@ -244,7 +246,7 @@ func (p *baseTxnContextProvider) ActivateTxn() (kv.Transaction, error) { txn.SetOption(kv.IsolationLevel, kv.RC) } - sessiontxn.SetTxnAssertionLevel(txn, sessVars.AssertionLevel) + internal.SetTxnAssertionLevel(txn, sessVars.AssertionLevel) if p.causalConsistencyOnly { txn.SetOption(kv.GuaranteeLinearizability, false) @@ -277,7 +279,7 @@ func (p *baseTxnContextProvider) prepareTxn() error { return p.prepareTxnWithTS(snapshotTS) } - future := sessiontxn.NewOracleFuture(p.ctx, p.sctx, p.sctx.GetSessionVars().TxnCtx.TxnScope) + future := newOracleFuture(p.ctx, p.sctx, p.sctx.GetSessionVars().TxnCtx.TxnScope) return p.replaceTxnTsFuture(future) } @@ -289,7 +291,7 @@ func (p *baseTxnContextProvider) prepareTxnWithOracleTS() error { return nil } - future := sessiontxn.NewOracleFuture(p.ctx, p.sctx, p.sctx.GetSessionVars().TxnCtx.TxnScope) + future := newOracleFuture(p.ctx, p.sctx, p.sctx.GetSessionVars().TxnCtx.TxnScope) return p.replaceTxnTsFuture(future) } @@ -375,7 +377,7 @@ func (p *baseTxnContextProvider) getSnapshotByTS(snapshotTS uint64) (kv.Snapshot } sessVars := p.sctx.GetSessionVars() - snapshot := sessiontxn.GetSnapshotWithTS(p.sctx, snapshotTS) + snapshot := internal.GetSnapshotWithTS(p.sctx, snapshotTS) replicaReadType := sessVars.GetReplicaRead() if replicaReadType.IsFollowerRead() && !sessVars.StmtCtx.RCCheckTS { @@ -384,3 +386,41 @@ func (p *baseTxnContextProvider) getSnapshotByTS(snapshotTS uint64) (kv.Snapshot return snapshot, nil } + +// canReuseTxnWhenExplicitBegin returns whether we should reuse the txn when starting a transaction explicitly +func canReuseTxnWhenExplicitBegin(sctx sessionctx.Context) bool { + sessVars := sctx.GetSessionVars() + txnCtx := sessVars.TxnCtx + // If BEGIN is the first statement in TxnCtx, we can reuse the existing transaction, without the + // need to call NewTxn, which commits the existing transaction and begins a new one. + // If the last un-committed/un-rollback transaction is a time-bounded read-only transaction, we should + // always create a new transaction. + // If the variable `tidb_snapshot` is set, we should always create a new transaction because the current txn may be + // initialized with snapshot ts. + return txnCtx.History == nil && !txnCtx.IsStaleness && sessVars.SnapshotTS == 0 +} + +// newOracleFuture creates new future according to the scope and the session context +func newOracleFuture(ctx context.Context, sctx sessionctx.Context, scope string) oracle.Future { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("isolation.newOracleFuture", opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } + + oracleStore := sctx.GetStore().GetOracle() + option := &oracle.Option{TxnScope: scope} + + if sctx.GetSessionVars().LowResolutionTSO { + return oracleStore.GetLowResolutionTimestampAsync(ctx, option) + } + return oracleStore.GetTimestampAsync(ctx, option) +} + +// funcFuture implements oracle.Future +type funcFuture func() (uint64, error) + +// Wait returns a ts got from the func +func (f funcFuture) Wait() (uint64, error) { + return f() +} diff --git a/sessiontxn/isolation/readcommitted.go b/sessiontxn/isolation/readcommitted.go index c24062a39f20e..d34d6c9405b1d 100644 --- a/sessiontxn/isolation/readcommitted.go +++ b/sessiontxn/isolation/readcommitted.go @@ -133,7 +133,7 @@ func (p *PessimisticRCTxnContextProvider) prepareStmtTS() { var stmtTSFuture oracle.Future switch { case p.stmtUseStartTS: - stmtTSFuture = sessiontxn.FuncFuture(p.getTxnStartTS) + stmtTSFuture = funcFuture(p.getTxnStartTS) case p.latestOracleTSValid && sessVars.StmtCtx.RCCheckTS: stmtTSFuture = sessiontxn.ConstantFuture(p.latestOracleTS) default: @@ -143,9 +143,9 @@ func (p *PessimisticRCTxnContextProvider) prepareStmtTS() { p.stmtTSFuture = stmtTSFuture } -func (p *PessimisticRCTxnContextProvider) getOracleFuture() sessiontxn.FuncFuture { +func (p *PessimisticRCTxnContextProvider) getOracleFuture() funcFuture { txnCtx := p.sctx.GetSessionVars().TxnCtx - future := sessiontxn.NewOracleFuture(p.ctx, p.sctx, txnCtx.TxnScope) + future := newOracleFuture(p.ctx, p.sctx, txnCtx.TxnScope) return func() (ts uint64, err error) { if ts, err = future.Wait(); err != nil { return diff --git a/sessiontxn/isolation/repeatable_read.go b/sessiontxn/isolation/repeatable_read.go index f09f9ca415e2a..2ea4bd41c5996 100644 --- a/sessiontxn/isolation/repeatable_read.go +++ b/sessiontxn/isolation/repeatable_read.go @@ -80,7 +80,7 @@ func (p *PessimisticRRTxnContextProvider) getForUpdateTs() (ts uint64, err error } txnCtx := p.sctx.GetSessionVars().TxnCtx - futureTS := sessiontxn.NewOracleFuture(p.ctx, p.sctx, txnCtx.TxnScope) + futureTS := newOracleFuture(p.ctx, p.sctx, txnCtx.TxnScope) if ts, err = futureTS.Wait(); err != nil { return 0, err diff --git a/sessiontxn/staleread/provider.go b/sessiontxn/staleread/provider.go index 31d02726526ae..b7ff889c0e29c 100644 --- a/sessiontxn/staleread/provider.go +++ b/sessiontxn/staleread/provider.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/sessiontxn" + "github.com/pingcap/tidb/sessiontxn/internal" "github.com/pingcap/tidb/table/temptable" ) @@ -89,7 +90,7 @@ func (p *StalenessTxnContextProvider) OnInitialize(ctx context.Context, tp sessi // with the staleness snapshot ts. After that, it sets the relevant context variables. func (p *StalenessTxnContextProvider) activateStaleTxn() error { var err error - if err = sessiontxn.CommitBeforeEnterNewTxn(p.ctx, p.sctx); err != nil { + if err = internal.CommitBeforeEnterNewTxn(p.ctx, p.sctx); err != nil { return err } @@ -108,7 +109,7 @@ func (p *StalenessTxnContextProvider) activateStaleTxn() error { txn.SetVars(sessVars.KVVars) txn.SetOption(kv.IsStalenessReadOnly, true) txn.SetOption(kv.TxnScope, txnScope) - sessiontxn.SetTxnAssertionLevel(txn, sessVars.AssertionLevel) + internal.SetTxnAssertionLevel(txn, sessVars.AssertionLevel) is, err := GetSessionSnapshotInfoSchema(p.sctx, p.ts) if err != nil { return errors.Trace(err) @@ -208,7 +209,7 @@ func (p *StalenessTxnContextProvider) GetSnapshotWithStmtReadTS() (kv.Snapshot, } sessVars := p.sctx.GetSessionVars() - snapshot := sessiontxn.GetSnapshotWithTS(p.sctx, p.ts) + snapshot := internal.GetSnapshotWithTS(p.sctx, p.ts) replicaReadType := sessVars.GetReplicaRead() if replicaReadType.IsFollowerRead() { diff --git a/sessiontxn/txn_manager_test.go b/sessiontxn/txn_manager_test.go index 983513fa44d03..94fb5cec60ab5 100644 --- a/sessiontxn/txn_manager_test.go +++ b/sessiontxn/txn_manager_test.go @@ -23,10 +23,14 @@ import ( "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessiontxn" + "github.com/pingcap/tidb/sessiontxn/internal" "github.com/pingcap/tidb/sessiontxn/staleread" + "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/testkit" + "github.com/pingcap/tidb/tests/realtikvtest" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" ) @@ -306,7 +310,7 @@ func TestGetSnapshot(t *testing.T) { check: func(t *testing.T, sctx sessionctx.Context) { ts, err := mgr.GetStmtReadTS() require.NoError(t, err) - compareSnap := sessiontxn.GetSnapshotWithTS(sctx, ts) + compareSnap := internal.GetSnapshotWithTS(sctx, ts) snap, err := mgr.GetSnapshotWithStmtReadTS() require.NoError(t, err) require.True(t, isSnapshotEqual(t, compareSnap, snap)) @@ -316,7 +320,7 @@ func TestGetSnapshot(t *testing.T) { tk.MustQuery("select * from t for update").Check(testkit.Rows("1", "3", "10")) ts, err = mgr.GetStmtForUpdateTS() require.NoError(t, err) - compareSnap2 := sessiontxn.GetSnapshotWithTS(sctx, ts) + compareSnap2 := internal.GetSnapshotWithTS(sctx, ts) snap, err = mgr.GetSnapshotWithStmtReadTS() require.NoError(t, err) require.False(t, isSnapshotEqual(t, compareSnap2, snap)) @@ -336,7 +340,7 @@ func TestGetSnapshot(t *testing.T) { check: func(t *testing.T, sctx sessionctx.Context) { ts, err := mgr.GetStmtReadTS() require.NoError(t, err) - compareSnap := sessiontxn.GetSnapshotWithTS(sctx, ts) + compareSnap := internal.GetSnapshotWithTS(sctx, ts) snap, err := mgr.GetSnapshotWithStmtReadTS() require.NoError(t, err) require.True(t, isSnapshotEqual(t, compareSnap, snap)) @@ -346,7 +350,7 @@ func TestGetSnapshot(t *testing.T) { tk.MustQuery("select * from t").Check(testkit.Rows("1", "3", "10")) ts, err = mgr.GetStmtForUpdateTS() require.NoError(t, err) - compareSnap2 := sessiontxn.GetSnapshotWithTS(sctx, ts) + compareSnap2 := internal.GetSnapshotWithTS(sctx, ts) snap, err = mgr.GetSnapshotWithStmtReadTS() require.NoError(t, err) require.True(t, isSnapshotEqual(t, compareSnap2, snap)) @@ -365,7 +369,7 @@ func TestGetSnapshot(t *testing.T) { check: func(t *testing.T, sctx sessionctx.Context) { ts, err := mgr.GetStmtReadTS() require.NoError(t, err) - compareSnap := sessiontxn.GetSnapshotWithTS(sctx, ts) + compareSnap := internal.GetSnapshotWithTS(sctx, ts) snap, err := mgr.GetSnapshotWithStmtReadTS() require.NoError(t, err) require.True(t, isSnapshotEqual(t, compareSnap, snap)) @@ -375,7 +379,7 @@ func TestGetSnapshot(t *testing.T) { tk.MustQuery("select * from t for update").Check(testkit.Rows("1", "3")) ts, err = mgr.GetStmtForUpdateTS() require.NoError(t, err) - compareSnap2 := sessiontxn.GetSnapshotWithTS(sctx, ts) + compareSnap2 := internal.GetSnapshotWithTS(sctx, ts) snap, err = mgr.GetSnapshotWithStmtReadTS() require.NoError(t, err) require.True(t, isSnapshotEqual(t, compareSnap2, snap)) @@ -396,7 +400,7 @@ func TestGetSnapshot(t *testing.T) { check: func(t *testing.T, sctx sessionctx.Context) { ts, err := mgr.GetStmtReadTS() require.NoError(t, err) - compareSnap := sessiontxn.GetSnapshotWithTS(sctx, ts) + compareSnap := internal.GetSnapshotWithTS(sctx, ts) snap, err := mgr.GetSnapshotWithStmtReadTS() require.NoError(t, err) require.True(t, isSnapshotEqual(t, compareSnap, snap)) @@ -406,7 +410,7 @@ func TestGetSnapshot(t *testing.T) { tk.MustQuery("select * from t for update").Check(testkit.Rows("1", "3")) ts, err = mgr.GetStmtForUpdateTS() require.NoError(t, err) - compareSnap2 := sessiontxn.GetSnapshotWithTS(sctx, ts) + compareSnap2 := internal.GetSnapshotWithTS(sctx, ts) snap, err = mgr.GetSnapshotWithStmtReadTS() require.NoError(t, err) require.True(t, isSnapshotEqual(t, compareSnap2, snap)) @@ -440,6 +444,65 @@ func TestGetSnapshot(t *testing.T) { } } +func TestSnapshotInterceptor(t *testing.T) { + store, clean := realtikvtest.CreateMockStoreAndSetup(t) + defer clean() + + tk := testkit.NewTestKit(t, store) + tk.MustExec("create temporary table test.tmp1 (id int primary key)") + tbl, err := tk.Session().GetDomainInfoSchema().(infoschema.InfoSchema).TableByName(model.NewCIStr("test"), model.NewCIStr("tmp1")) + require.NoError(t, err) + require.Equal(t, model.TempTableLocal, tbl.Meta().TempTableType) + tblID := tbl.Meta().ID + + // prepare a kv pair for temporary table + k := append(tablecodec.EncodeTablePrefix(tblID), 1) + require.NoError(t, tk.Session().GetSessionVars().TemporaryTableData.SetTableKey(tblID, k, []byte("v1"))) + + initTxnFuncs := []func() error{ + func() error { + err := tk.Session().PrepareTxnCtx(context.TODO()) + if err == nil { + err = sessiontxn.GetTxnManager(tk.Session()).AdviseWarmup() + } + return err + }, + func() error { + return sessiontxn.NewTxn(context.Background(), tk.Session()) + }, + func() error { + return sessiontxn.GetTxnManager(tk.Session()).EnterNewTxn(context.TODO(), &sessiontxn.EnterNewTxnRequest{ + Type: sessiontxn.EnterNewTxnWithBeginStmt, + StaleReadTS: 0, + }) + }, + } + + for _, initFunc := range initTxnFuncs { + require.NoError(t, initFunc()) + + require.NoError(t, sessiontxn.GetTxnManager(tk.Session()).OnStmtStart(context.TODO(), nil)) + txn, err := tk.Session().Txn(true) + require.NoError(t, err) + + val, err := txn.Get(context.Background(), k) + require.NoError(t, err) + require.Equal(t, []byte("v1"), val) + + val, err = txn.GetSnapshot().Get(context.Background(), k) + require.NoError(t, err) + require.Equal(t, []byte("v1"), val) + + tk.Session().RollbackTxn(context.Background()) + } + + // Also check GetSnapshotWithTS + snap := internal.GetSnapshotWithTS(tk.Session(), 0) + val, err := snap.Get(context.Background(), k) + require.NoError(t, err) + require.Equal(t, []byte("v1"), val) +} + func checkBasicActiveTxn(t *testing.T, sctx sessionctx.Context) kv.Transaction { txn, err := sctx.Txn(false) require.NoError(t, err) diff --git a/tests/realtikvtest/sessiontest/temporary_table_test.go b/tests/realtikvtest/sessiontest/temporary_table_test.go index 67f94d381fbf7..c669ac199544c 100644 --- a/tests/realtikvtest/sessiontest/temporary_table_test.go +++ b/tests/realtikvtest/sessiontest/temporary_table_test.go @@ -15,7 +15,6 @@ package sessiontest import ( - "context" "fmt" "sort" "strconv" @@ -23,14 +22,10 @@ import ( "testing" "github.com/pingcap/tidb/errno" - "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/terror" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx/variable" - "github.com/pingcap/tidb/sessiontxn" - "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/tests/realtikvtest" "github.com/stretchr/testify/require" @@ -284,65 +279,6 @@ func TestLocalTemporaryTableUpdate(t *testing.T) { } } -func TestTemporaryTableInterceptor(t *testing.T) { - store, clean := realtikvtest.CreateMockStoreAndSetup(t) - defer clean() - - tk := testkit.NewTestKit(t, store) - tk.MustExec("create temporary table test.tmp1 (id int primary key)") - tbl, err := tk.Session().GetInfoSchema().(infoschema.InfoSchema).TableByName(model.NewCIStr("test"), model.NewCIStr("tmp1")) - require.NoError(t, err) - require.Equal(t, model.TempTableLocal, tbl.Meta().TempTableType) - tblID := tbl.Meta().ID - - // prepare a kv pair for temporary table - k := append(tablecodec.EncodeTablePrefix(tblID), 1) - require.NoError(t, tk.Session().GetSessionVars().TemporaryTableData.SetTableKey(tblID, k, []byte("v1"))) - - initTxnFuncs := []func() error{ - func() error { - err := tk.Session().PrepareTxnCtx(context.TODO()) - if err == nil { - err = sessiontxn.GetTxnManager(tk.Session()).AdviseWarmup() - } - return err - }, - func() error { - return sessiontxn.NewTxn(context.Background(), tk.Session()) - }, - func() error { - return sessiontxn.GetTxnManager(tk.Session()).EnterNewTxn(context.TODO(), &sessiontxn.EnterNewTxnRequest{ - Type: sessiontxn.EnterNewTxnWithBeginStmt, - StaleReadTS: 0, - }) - }, - } - - for _, initFunc := range initTxnFuncs { - require.NoError(t, initFunc()) - - require.NoError(t, sessiontxn.GetTxnManager(tk.Session()).OnStmtStart(context.TODO(), nil)) - txn, err := tk.Session().Txn(true) - require.NoError(t, err) - - val, err := txn.Get(context.Background(), k) - require.NoError(t, err) - require.Equal(t, []byte("v1"), val) - - val, err = txn.GetSnapshot().Get(context.Background(), k) - require.NoError(t, err) - require.Equal(t, []byte("v1"), val) - - tk.Session().RollbackTxn(context.Background()) - } - - // Also check GetSnapshotWithTS - snap := sessiontxn.GetSnapshotWithTS(tk.Session(), 0) - val, err := snap.Get(context.Background(), k) - require.NoError(t, err) - require.Equal(t, []byte("v1"), val) -} - func TestTemporaryTableSize(t *testing.T) { // Test the @@tidb_tmp_table_max_size system variable. From 3e8f382f6ce5d66b2286e4cc228492ff67a09375 Mon Sep 17 00:00:00 2001 From: Chengpeng Yan <41809508+Reminiscent@users.noreply.github.com> Date: Wed, 13 Jul 2022 13:35:05 +0800 Subject: [PATCH 07/27] planner: decorrelate the APPLY when the inner's projection is empty (#36073) close pingcap/tidb#35985 --- planner/core/rule_decorrelate.go | 4 +++- planner/core/testdata/plan_suite_unexported_in.json | 3 ++- planner/core/testdata/plan_suite_unexported_out.json | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/planner/core/rule_decorrelate.go b/planner/core/rule_decorrelate.go index 7626f5863f2c2..a09c55f1dac3e 100644 --- a/planner/core/rule_decorrelate.go +++ b/planner/core/rule_decorrelate.go @@ -154,7 +154,9 @@ func (s *decorrelateSolver) optimize(ctx context.Context, p LogicalPlan, opt *lo return s.optimize(ctx, p, opt) } } else if proj, ok := innerPlan.(*LogicalProjection); ok { - allConst := true + // After the column pruning, some expressions in the projection operator may be pruned. + // In this situation, we can decorrelate the apply operator. + allConst := len(proj.Exprs) > 0 for _, expr := range proj.Exprs { if len(expression.ExtractCorColumns(expr)) > 0 || !expression.ExtractColumnSet(expr).IsEmpty() { allConst = false diff --git a/planner/core/testdata/plan_suite_unexported_in.json b/planner/core/testdata/plan_suite_unexported_in.json index 8600fc0c48d0f..0e68e3cd27deb 100644 --- a/planner/core/testdata/plan_suite_unexported_in.json +++ b/planner/core/testdata/plan_suite_unexported_in.json @@ -139,7 +139,8 @@ "select t1.b from t t1 where exists(select t2.b from t t2 where t2.a = t1.a order by t2.a)", // `Sort` will not be eliminated, if it is not the top level operator. "select t1.b from t t1 where t1.b = (select t2.b from t t2 where t2.a = t1.a order by t2.a limit 1)", - "select (select 1 from t t1 where t1.a = t2.a) from t t2" + "select (select 1 from t t1 where t1.a = t2.a) from t t2", + "select count(1) from (select (select count(t1.a) as a from t t1 where t1.c = t2.c) as a from t t2) as t3" ] }, { diff --git a/planner/core/testdata/plan_suite_unexported_out.json b/planner/core/testdata/plan_suite_unexported_out.json index b116b4ffdfe62..562fcacf94547 100644 --- a/planner/core/testdata/plan_suite_unexported_out.json +++ b/planner/core/testdata/plan_suite_unexported_out.json @@ -125,7 +125,8 @@ "Join{DataScan(t1)->DataScan(t2)}(test.t.a,test.t.a)(test.t.b,test.t.b)->Projection", "Join{DataScan(t1)->DataScan(t2)}(test.t.a,test.t.a)->Projection", "Apply{DataScan(t1)->DataScan(t2)->Sel([eq(test.t.a, test.t.a)])->Projection->Sort->Limit}->Projection->Sel([eq(test.t.b, test.t.b)])->Projection", - "Apply{DataScan(t2)->DataScan(t1)->Sel([eq(test.t.a, test.t.a)])->Projection}->Projection" + "Apply{DataScan(t2)->DataScan(t1)->Sel([eq(test.t.a, test.t.a)])->Projection}->Projection", + "Join{DataScan(t2)->DataScan(t1)->Aggr(firstrow(test.t.c),count(1))}(test.t.c,test.t.c)->Projection->Aggr(count(1))->Projection" ] }, { From 8f83f4f88ed720592054db1c0a122a56069471e5 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 13 Jul 2022 14:05:05 +0800 Subject: [PATCH 08/27] telemetry: add reviewer rule (#36084) close pingcap/tidb#36087 --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ccc42ebaa1ee0..b1d1d17e8acd9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,3 +2,4 @@ /sessionctx/variable @pingcap/tidb-configuration-reviewer /config/config.toml.example @pingcap/tidb-configuration-reviewer /session/bootstrap.go @pingcap/tidb-configuration-reviewer +/telemetry/ @pingcap/telemetry-reviewer From 50437e1d40875ac69b24596eeae032e4690d192a Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 13 Jul 2022 14:27:05 +0800 Subject: [PATCH 09/27] *: bazel use jdk 17 (#36070) --- .bazelrc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.bazelrc b/.bazelrc index 855bc5770e169..2a1b030700ec2 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,6 +1,12 @@ startup --host_jvm_args=-Xmx8g startup --unlimit_coredumps +build --java_language_version=17 +build --java_runtime_version=17 +build --tool_java_language_version=17 +build --tool_java_runtime_version=17 +build --experimental_remote_cache_compression + run --color=yes build:release --workspace_status_command=./build/print-workspace-status.sh --stamp build:release --config=ci From 0e13d5d00990e0d573dbe20069f7e69d991d078b Mon Sep 17 00:00:00 2001 From: lance6716 Date: Wed, 13 Jul 2022 14:57:05 +0800 Subject: [PATCH 10/27] ddl: implement table granularity DDL for SchemaTracker (#36077) ref pingcap/tidb#35933 --- ddl/db_integration_test.go | 14 +- ddl/db_partition_test.go | 47 ++++++- ddl/db_rename_test.go | 37 +++++- ddl/db_table_test.go | 60 ++++++++- ddl/db_test.go | 7 +- ddl/ddl_api.go | 44 ++++--- ddl/mock.go | 2 +- ddl/schematracker/checker.go | 137 +++++++++++++++++--- ddl/schematracker/dm_tracker.go | 187 ++++++++++++++++++++++++++- ddl/schematracker/dm_tracker_test.go | 82 ++++++++++++ executor/ddl_test.go | 21 ++- 11 files changed, 578 insertions(+), 60 deletions(-) create mode 100644 ddl/schematracker/dm_tracker_test.go diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index 97ca00beed15b..4b1b510bfe1db 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -149,8 +149,13 @@ func TestInvalidNameWhenCreateTable(t *testing.T) { // TestCreateTableIfNotExistsLike for issue #6879 func TestCreateTableIfNotExistsLike(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("USE test;") @@ -176,8 +181,13 @@ func TestCreateTableIfNotExistsLike(t *testing.T) { // for issue #9910 func TestCreateTableWithKeyWord(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("USE test;") diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index 41a3ddc3f3a7d..b112e85c8eb50 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/ddl/schematracker" "github.com/pingcap/tidb/ddl/testutil" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/errno" @@ -83,8 +84,13 @@ func checkGlobalIndexCleanUpDone(t *testing.T, ctx sessionctx.Context, tblInfo * } func TestCreateTableWithPartition(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test;") tk.MustExec("drop table if exists tp;") @@ -336,8 +342,13 @@ partition by range (a) } func TestCreateTableWithHashPartition(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test;") tk.MustExec("drop table if exists employees;") @@ -395,8 +406,13 @@ func TestCreateTableWithHashPartition(t *testing.T) { } func TestCreateTableWithRangeColumnPartition(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test;") tk.MustExec("drop table if exists log_message_1;") @@ -637,6 +653,10 @@ create table log_message_1 ( tk.MustExec("drop table if exists t;") tk.MustExec(`create table t(a binary) partition by range columns (a) (partition p0 values less than (X'0C'));`) + + // TODO: we haven't implement AlterTable in SchemaTracker yet + ddlChecker.Disable() + tk.MustExec(`alter table t add partition (partition p1 values less than (X'0D'), partition p2 values less than (X'0E'));`) tk.MustExec(`insert into t values (X'0B'), (X'0C'), (X'0D')`) tk.MustQuery(`select * from t where a < X'0D' order by a`).Check(testkit.Rows("\x0B", "\x0C")) @@ -785,8 +805,13 @@ func generatePartitionTableByNum(num int) string { } func TestCreateTableWithListPartition(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test;") tk.MustExec("set @@session.tidb_enable_list_partition = ON") @@ -932,8 +957,13 @@ func TestCreateTableWithListPartition(t *testing.T) { } func TestCreateTableWithListColumnsPartition(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test;") tk.MustExec("set @@session.tidb_enable_list_partition = ON") @@ -1467,8 +1497,13 @@ func TestAlterTableTruncatePartitionByListColumns(t *testing.T) { } func TestCreateTableWithKeyPartition(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test;") tk.MustExec("drop table if exists tm1;") diff --git a/ddl/db_rename_test.go b/ddl/db_rename_test.go index 5017beb99615c..bacba15d1a602 100644 --- a/ddl/db_rename_test.go +++ b/ddl/db_rename_test.go @@ -19,8 +19,10 @@ import ( "testing" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/ddl/schematracker" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/errno" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/testkit" "github.com/stretchr/testify/require" @@ -58,8 +60,14 @@ func TestRenameTableWithLocked(t *testing.T) { config.UpdateGlobal(func(conf *config.Config) { conf.EnableTableLock = true }) - store, clean := testkit.CreateMockStore(t) + + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("create database renamedb") tk.MustExec("create database renamedb2") @@ -93,8 +101,24 @@ func TestAlterTableRenameTable(t *testing.T) { } func renameTableTest(t *testing.T, sql string, isAlterTable bool) { - store, clean := testkit.CreateMockStore(t) - defer clean() + var ( + store kv.Storage + clean func() + ) + + if isAlterTable { + store, clean = testkit.CreateMockStore(t) + defer clean() + } else { + var dom *domain.Domain + store, dom, clean = testkit.CreateMockStoreAndDomain(t) + defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + } + tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustGetErrCode("rename table tb1 to tb2;", errno.ErrNoSuchTable) @@ -197,8 +221,13 @@ func renameTableTest(t *testing.T, sql string, isAlterTable bool) { } func TestRenameMultiTables(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("create table t1(id int)") diff --git a/ddl/db_table_test.go b/ddl/db_table_test.go index a385acd7e3427..b2550e4060555 100644 --- a/ddl/db_table_test.go +++ b/ddl/db_table_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/ddl/schematracker" testddlutil "github.com/pingcap/tidb/ddl/testutil" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/errno" @@ -227,8 +228,13 @@ func TestTransactionOnAddDropColumn(t *testing.T) { } func TestCreateTableWithSetCol(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("create table t_set (a int, b set('e') default '');") @@ -284,8 +290,13 @@ func TestCreateTableWithSetCol(t *testing.T) { } func TestCreateTableWithEnumCol(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test") // It's for failure cases. @@ -316,8 +327,13 @@ func TestCreateTableWithEnumCol(t *testing.T) { } func TestCreateTableWithIntegerColWithDefault(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test") // It's for failure cases. @@ -913,3 +929,41 @@ func TestAddColumn2(t *testing.T) { re.Check(testkit.Rows("1 2")) tk.MustQuery("select a,b,_tidb_rowid from t2").Check(testkit.Rows("1 3 2")) } + +func TestDropTables(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1;") + + failedSQL := "drop table t1;" + tk.MustGetErrCode(failedSQL, errno.ErrBadTable) + failedSQL = "drop table test2.t1;" + tk.MustGetErrCode(failedSQL, errno.ErrBadTable) + + tk.MustExec("create table t1 (a int);") + tk.MustExec("drop table if exists t1, t2;") + + tk.MustExec("create table t1 (a int);") + tk.MustExec("drop table if exists t2, t1;") + + // Without IF EXISTS, the statement drops all named tables that do exist, and returns an error indicating which + // nonexisting tables it was unable to drop. + // https://dev.mysql.com/doc/refman/5.7/en/drop-table.html + tk.MustExec("create table t1 (a int);") + failedSQL = "drop table t1, t2;" + tk.MustGetErrCode(failedSQL, errno.ErrBadTable) + + tk.MustExec("create table t1 (a int);") + failedSQL = "drop table t2, t1;" + tk.MustGetErrCode(failedSQL, errno.ErrBadTable) + + failedSQL = "show create table t1;" + tk.MustGetErrCode(failedSQL, errno.ErrNoSuchTable) +} diff --git a/ddl/db_test.go b/ddl/db_test.go index 8f2c6ac06a9fa..06d2f81b74c3a 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/ddl/schematracker" ddlutil "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/errno" @@ -536,9 +537,13 @@ func TestAddConstraintCheck(t *testing.T) { } func TestCreateTableIgnoreCheckConstraint(t *testing.T) { - store, clean := testkit.CreateMockStoreWithSchemaLease(t, dbTestLease) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("drop table if exists table_constraint_check") diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index c0a1a9fb2e48b..96348445a9320 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -1782,13 +1782,15 @@ func convertAutoRandomBitsToUnsigned(autoRandomBits int) (uint64, error) { return uint64(autoRandomBits), nil } -func buildTableInfo( +// BuildTableInfo creates a TableInfo. +func BuildTableInfo( ctx sessionctx.Context, tableName model.CIStr, cols []*table.Column, constraints []*ast.Constraint, charset string, - collate string) (tbInfo *model.TableInfo, err error) { + collate string, +) (tbInfo *model.TableInfo, err error) { tbInfo = &model.TableInfo{ Name: tableName, Version: model.CurrLatestTableInfoVersion, @@ -1992,6 +1994,11 @@ func checkTableInfoValidExtra(tbInfo *model.TableInfo) error { return err } +// CheckTableInfoValidWithStmt exposes checkTableInfoValidWithStmt to SchemaTracker. Maybe one day we can delete it. +func CheckTableInfoValidWithStmt(ctx sessionctx.Context, tbInfo *model.TableInfo, s *ast.CreateTableStmt) (err error) { + return checkTableInfoValidWithStmt(ctx, tbInfo, s) +} + func checkTableInfoValidWithStmt(ctx sessionctx.Context, tbInfo *model.TableInfo, s *ast.CreateTableStmt) (err error) { // All of these rely on the AST structure of expressions, which were // lost in the model (got serialized into strings). @@ -2050,7 +2057,8 @@ func checkTableInfoValid(tblInfo *model.TableInfo) error { return checkInvisibleIndexOnPK(tblInfo) } -func buildTableInfoWithLike(ctx sessionctx.Context, ident ast.Ident, referTblInfo *model.TableInfo, s *ast.CreateTableStmt) (*model.TableInfo, error) { +// BuildTableInfoWithLike builds a new table info according to CREATE TABLE ... LIKE statement. +func BuildTableInfoWithLike(ctx sessionctx.Context, ident ast.Ident, referTblInfo *model.TableInfo, s *ast.CreateTableStmt) (*model.TableInfo, error) { // Check the referred table is a real table object. if referTblInfo.IsSequence() || referTblInfo.IsView() { return nil, dbterror.ErrWrongObject.GenWithStackByArgs(ident.Schema, referTblInfo.Name, "BASE TABLE") @@ -2100,7 +2108,7 @@ func BuildTableInfoFromAST(s *ast.CreateTableStmt) (*model.TableInfo, error) { // buildTableInfoWithCheck builds model.TableInfo from a SQL statement. // Note: TableID and PartitionIDs are left as uninitialized value. func buildTableInfoWithCheck(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCharset, dbCollate string, placementPolicyRef *model.PolicyRefInfo) (*model.TableInfo, error) { - tbInfo, err := buildTableInfoWithStmt(ctx, s, dbCharset, dbCollate, placementPolicyRef) + tbInfo, err := BuildTableInfoWithStmt(ctx, s, dbCharset, dbCollate, placementPolicyRef) if err != nil { return nil, err } @@ -2133,15 +2141,15 @@ func BuildSessionTemporaryTableInfo(ctx sessionctx.Context, is infoschema.InfoSc if err != nil { return nil, infoschema.ErrTableNotExists.GenWithStackByArgs(referIdent.Schema, referIdent.Name) } - tbInfo, err = buildTableInfoWithLike(ctx, ident, referTbl.Meta(), s) + tbInfo, err = BuildTableInfoWithLike(ctx, ident, referTbl.Meta(), s) } else { tbInfo, err = buildTableInfoWithCheck(ctx, s, dbCharset, dbCollate, placementPolicyRef) } return tbInfo, err } -// buildTableInfoWithStmt builds model.TableInfo from a SQL statement without validity check -func buildTableInfoWithStmt(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCharset, dbCollate string, placementPolicyRef *model.PolicyRefInfo) (*model.TableInfo, error) { +// BuildTableInfoWithStmt builds model.TableInfo from a SQL statement without validity check +func BuildTableInfoWithStmt(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCharset, dbCollate string, placementPolicyRef *model.PolicyRefInfo) (*model.TableInfo, error) { colDefs := s.Cols tableCharset, tableCollate, err := getCharsetAndCollateInTableOption(0, s.Options) if err != nil { @@ -2166,7 +2174,7 @@ func buildTableInfoWithStmt(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCh } var tbInfo *model.TableInfo - tbInfo, err = buildTableInfo(ctx, s.Table.Name, cols, newConstraints, tableCharset, tableCollate) + tbInfo, err = BuildTableInfo(ctx, s.Table.Name, cols, newConstraints, tableCharset, tableCollate) if err != nil { return nil, errors.Trace(err) } @@ -2244,9 +2252,9 @@ func (d *ddl) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) (err e // build tableInfo var tbInfo *model.TableInfo if s.ReferTable != nil { - tbInfo, err = buildTableInfoWithLike(ctx, ident, referTbl.Meta(), s) + tbInfo, err = BuildTableInfoWithLike(ctx, ident, referTbl.Meta(), s) } else { - tbInfo, err = buildTableInfoWithStmt(ctx, s, schema.Charset, schema.Collate, schema.PlacementPolicyRef) + tbInfo, err = BuildTableInfoWithStmt(ctx, s, schema.Charset, schema.Collate, schema.PlacementPolicyRef) } if err != nil { return errors.Trace(err) @@ -2624,7 +2632,7 @@ func (d *ddl) RecoverTable(ctx sessionctx.Context, recoverInfo *RecoverInfo) (er } func (d *ddl) CreateView(ctx sessionctx.Context, s *ast.CreateViewStmt) (err error) { - viewInfo, err := buildViewInfo(ctx, s) + viewInfo, err := BuildViewInfo(ctx, s) if err != nil { return err } @@ -2648,7 +2656,7 @@ func (d *ddl) CreateView(ctx sessionctx.Context, s *ast.CreateViewStmt) (err err tblCollate = v } - tbInfo, err := buildTableInfo(ctx, s.ViewName.Name, cols, nil, tblCharset, tblCollate) + tbInfo, err := BuildTableInfo(ctx, s.ViewName.Name, cols, nil, tblCharset, tblCollate) if err != nil { return err } @@ -2662,7 +2670,8 @@ func (d *ddl) CreateView(ctx sessionctx.Context, s *ast.CreateViewStmt) (err err return d.CreateTableWithInfo(ctx, s.ViewName.Schema, tbInfo, onExist) } -func buildViewInfo(ctx sessionctx.Context, s *ast.CreateViewStmt) (*model.ViewInfo, error) { +// BuildViewInfo builds a ViewInfo structure from an ast.CreateViewStmt. +func BuildViewInfo(ctx sessionctx.Context, s *ast.CreateViewStmt) (*model.ViewInfo, error) { // Always Use `format.RestoreNameBackQuotes` to restore `SELECT` statement despite the `ANSI_QUOTES` SQL Mode is enabled or not. restoreFlag := format.RestoreStringSingleQuotes | format.RestoreKeyWordUppercase | format.RestoreNameBackQuotes var sb strings.Builder @@ -5435,7 +5444,7 @@ func (d *ddl) RenameTable(ctx sessionctx.Context, s *ast.RenameTableStmt) error func (d *ddl) renameTable(ctx sessionctx.Context, oldIdent, newIdent ast.Ident, isAlterTable bool) error { is := d.GetInfoSchemaWithInterceptor(ctx) tables := make(map[string]int64) - schemas, tableID, err := extractTblInfos(is, oldIdent, newIdent, isAlterTable, tables) + schemas, tableID, err := ExtractTblInfos(is, oldIdent, newIdent, isAlterTable, tables) if err != nil { return err } @@ -5480,7 +5489,7 @@ func (d *ddl) renameTables(ctx sessionctx.Context, oldIdents, newIdents []ast.Id tables := make(map[string]int64) for i := 0; i < len(oldIdents); i++ { - schemas, tableID, err = extractTblInfos(is, oldIdents[i], newIdents[i], isAlterTable, tables) + schemas, tableID, err = ExtractTblInfos(is, oldIdents[i], newIdents[i], isAlterTable, tables) if err != nil { return err } @@ -5513,7 +5522,8 @@ func (d *ddl) renameTables(ctx sessionctx.Context, oldIdents, newIdents []ast.Id return errors.Trace(err) } -func extractTblInfos(is infoschema.InfoSchema, oldIdent, newIdent ast.Ident, isAlterTable bool, tables map[string]int64) ([]*model.DBInfo, int64, error) { +// ExtractTblInfos extracts the table information from the infoschema. +func ExtractTblInfos(is infoschema.InfoSchema, oldIdent, newIdent ast.Ident, isAlterTable bool, tables map[string]int64) ([]*model.DBInfo, int64, error) { oldSchema, ok := is.SchemaByName(oldIdent.Schema) if !ok { if isAlterTable { @@ -6564,7 +6574,7 @@ func (d *ddl) CreateSequence(ctx sessionctx.Context, stmt *ast.CreateSequenceStm return err } // TiDB describe the sequence within a tableInfo, as a same-level object of a table and view. - tbInfo, err := buildTableInfo(ctx, ident.Name, nil, nil, "", "") + tbInfo, err := BuildTableInfo(ctx, ident.Name, nil, nil, "", "") if err != nil { return err } diff --git a/ddl/mock.go b/ddl/mock.go index 7f470cc979a7f..c5f1f69f086f9 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -159,7 +159,7 @@ func MockTableInfo(ctx sessionctx.Context, stmt *ast.CreateTableStmt, tableID in if err != nil { return nil, errors.Trace(err) } - tbl, err := buildTableInfo(ctx, stmt.Table.Name, cols, newConstraints, "", "") + tbl, err := BuildTableInfo(ctx, stmt.Table.Name, cols, newConstraints, "", "") if err != nil { return nil, errors.Trace(err) } diff --git a/ddl/schematracker/checker.go b/ddl/schematracker/checker.go index 7bc946e6b89d4..6103cc12a84c3 100644 --- a/ddl/schematracker/checker.go +++ b/ddl/schematracker/checker.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/owner" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/model" @@ -62,6 +63,11 @@ func (d *Checker) Enable() { d.closed = false } +// CreateTestDB creates a `test` database like the default behaviour of TiDB. +func (d Checker) CreateTestDB() { + d.tracker.createTestDB() +} + func (d Checker) checkDBInfo(ctx sessionctx.Context, dbName model.CIStr) { if d.closed { return @@ -69,6 +75,14 @@ func (d Checker) checkDBInfo(ctx sessionctx.Context, dbName model.CIStr) { dbInfo, _ := d.realDDL.GetInfoSchemaWithInterceptor(ctx).SchemaByName(dbName) dbInfo2 := d.tracker.SchemaByName(dbName) + if dbInfo == nil || dbInfo2 == nil { + if dbInfo == nil && dbInfo2 == nil { + return + } + errStr := fmt.Sprintf("inconsistent dbInfo, dbName: %s, real ddl: %p, schematracker：%p", dbName, dbInfo, dbInfo2) + panic(errStr) + } + result := bytes.NewBuffer(make([]byte, 0, 512)) err := executor.ConstructResultOfShowCreateDatabase(ctx, dbInfo, false, result) if err != nil { @@ -87,6 +101,41 @@ func (d Checker) checkDBInfo(ctx sessionctx.Context, dbName model.CIStr) { } } +func (d Checker) checkTableInfo(ctx sessionctx.Context, dbName, tableName model.CIStr) { + if d.closed { + return + } + + tableInfo, _ := d.realDDL.GetInfoSchemaWithInterceptor(ctx).TableByName(dbName, tableName) + tableInfo2, _ := d.tracker.TableByName(dbName, tableName) + + if tableInfo == nil || tableInfo2 == nil { + if tableInfo == nil && tableInfo2 == nil { + return + } + errStr := fmt.Sprintf("inconsistent tableInfo, dbName: %s, tableName: %s, real ddl: %p, schematracker：%p", + dbName, tableName, tableInfo, tableInfo2) + panic(errStr) + } + + result := bytes.NewBuffer(make([]byte, 0, 512)) + err := executor.ConstructResultOfShowCreateTable(ctx, tableInfo.Meta(), autoid.Allocators{}, result) + if err != nil { + panic(err) + } + result2 := bytes.NewBuffer(make([]byte, 0, 512)) + err = executor.ConstructResultOfShowCreateTable(ctx, tableInfo2, autoid.Allocators{}, result2) + if err != nil { + panic(err) + } + s1 := result.String() + s2 := result2.String() + if s1 != s2 { + errStr := fmt.Sprintf("%s != %s", s1, s2) + panic(errStr) + } +} + // CreateSchema implements the DDL interface. func (d Checker) CreateSchema(ctx sessionctx.Context, stmt *ast.CreateDatabaseStmt) error { err := d.realDDL.CreateSchema(ctx, stmt) @@ -127,25 +176,53 @@ func (d Checker) DropSchema(ctx sessionctx.Context, stmt *ast.DropDatabaseStmt) if err != nil { panic(err) } + + d.checkDBInfo(ctx, stmt.Name) return nil } // CreateTable implements the DDL interface. func (d Checker) CreateTable(ctx sessionctx.Context, stmt *ast.CreateTableStmt) error { - //TODO implement me - panic("implement me") + err := d.realDDL.CreateTable(ctx, stmt) + if err != nil { + return err + } + // some unit test will also check warnings, we reset the warnings after SchemaTracker use session context again. + count := ctx.GetSessionVars().StmtCtx.WarningCount() + err = d.tracker.CreateTable(ctx, stmt) + if err != nil { + panic(err) + } + ctx.GetSessionVars().StmtCtx.TruncateWarnings(int(count)) + + d.checkTableInfo(ctx, stmt.Table.Schema, stmt.Table.Name) + return nil } // CreateView implements the DDL interface. func (d Checker) CreateView(ctx sessionctx.Context, stmt *ast.CreateViewStmt) error { - //TODO implement me - panic("implement me") + err := d.realDDL.CreateView(ctx, stmt) + if err != nil { + return err + } + err = d.tracker.CreateView(ctx, stmt) + if err != nil { + panic(err) + } + + d.checkTableInfo(ctx, stmt.ViewName.Schema, stmt.ViewName.Name) + return nil } // DropTable implements the DDL interface. func (d Checker) DropTable(ctx sessionctx.Context, stmt *ast.DropTableStmt) (err error) { - //TODO implement me - panic("implement me") + err = d.realDDL.DropTable(ctx, stmt) + _ = d.tracker.DropTable(ctx, stmt) + + for _, tableName := range stmt.Tables { + d.checkTableInfo(ctx, tableName.Schema, tableName.Name) + } + return err } // RecoverTable implements the DDL interface. @@ -156,8 +233,19 @@ func (d Checker) RecoverTable(ctx sessionctx.Context, recoverInfo *ddl.RecoverIn // DropView implements the DDL interface. func (d Checker) DropView(ctx sessionctx.Context, stmt *ast.DropTableStmt) (err error) { - //TODO implement me - panic("implement me") + err = d.realDDL.DropView(ctx, stmt) + if err != nil { + return err + } + err = d.tracker.DropView(ctx, stmt) + if err != nil { + panic(err) + } + + for _, tableName := range stmt.Tables { + d.checkTableInfo(ctx, tableName.Schema, tableName.Name) + } + return nil } // CreateIndex implements the DDL interface. @@ -174,7 +262,13 @@ func (d Checker) DropIndex(ctx sessionctx.Context, stmt *ast.DropIndexStmt) erro // AlterTable implements the DDL interface. func (d Checker) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast.AlterTableStmt) error { - //TODO implement me + err := d.realDDL.AlterTable(ctx, sctx, stmt) + if err != nil { + return err + } + if d.closed { + return nil + } panic("implement me") } @@ -186,26 +280,35 @@ func (d Checker) TruncateTable(ctx sessionctx.Context, tableIdent ast.Ident) err // RenameTable implements the DDL interface. func (d Checker) RenameTable(ctx sessionctx.Context, stmt *ast.RenameTableStmt) error { - //TODO implement me - panic("implement me") + err := d.realDDL.RenameTable(ctx, stmt) + if err != nil { + return err + } + err = d.tracker.RenameTable(ctx, stmt) + if err != nil { + panic(err) + } + + for _, tableName := range stmt.TableToTables { + d.checkTableInfo(ctx, tableName.OldTable.Schema, tableName.OldTable.Name) + d.checkTableInfo(ctx, tableName.NewTable.Schema, tableName.NewTable.Name) + } + return nil } // LockTables implements the DDL interface. func (d Checker) LockTables(ctx sessionctx.Context, stmt *ast.LockTablesStmt) error { - //TODO implement me - panic("implement me") + return d.realDDL.LockTables(ctx, stmt) } // UnlockTables implements the DDL interface. func (d Checker) UnlockTables(ctx sessionctx.Context, lockedTables []model.TableLockTpInfo) error { - //TODO implement me - panic("implement me") + return d.realDDL.UnlockTables(ctx, lockedTables) } // CleanupTableLock implements the DDL interface. func (d Checker) CleanupTableLock(ctx sessionctx.Context, tables []*ast.TableName) error { - //TODO implement me - panic("implement me") + return d.realDDL.CleanupTableLock(ctx, tables) } // UpdateTableReplicaInfo implements the DDL interface. diff --git a/ddl/schematracker/dm_tracker.go b/ddl/schematracker/dm_tracker.go index 1a0365a6f1778..0391fafc39c30 100644 --- a/ddl/schematracker/dm_tracker.go +++ b/ddl/schematracker/dm_tracker.go @@ -20,6 +20,7 @@ package schematracker import ( "context" + "strings" "time" "github.com/ngaut/pools" @@ -82,6 +83,12 @@ func (d SchemaTracker) CreateSchema(ctx sessionctx.Context, stmt *ast.CreateData return d.CreateSchemaWithInfo(ctx, dbInfo, onExist) } +func (d SchemaTracker) createTestDB() { + _ = d.CreateSchema(nil, &ast.CreateDatabaseStmt{ + Name: model.NewCIStr("test"), + }) +} + // CreateSchemaWithInfo implements the DDL interface. func (d SchemaTracker) CreateSchemaWithInfo(ctx sessionctx.Context, dbInfo *model.DBInfo, onExist ddl.OnExist) error { oldInfo := d.SchemaByName(dbInfo.Name) @@ -156,7 +163,51 @@ func (d SchemaTracker) DropSchema(ctx sessionctx.Context, stmt *ast.DropDatabase // CreateTable implements the DDL interface. func (d SchemaTracker) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) error { - panic("not implemented") + ident := ast.Ident{Schema: s.Table.Schema, Name: s.Table.Name} + schema := d.SchemaByName(ident.Schema) + if schema == nil { + return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(ident.Schema) + } + + // suppress ErrTooLongKey + ctx.GetSessionVars().StrictSQLMode = false + + var ( + referTbl *model.TableInfo + err error + ) + if s.ReferTable != nil { + referTbl, err = d.TableByName(s.ReferTable.Schema, s.ReferTable.Name) + if err != nil { + return infoschema.ErrTableNotExists.GenWithStackByArgs(s.ReferTable.Schema, s.ReferTable.Name) + } + } + + // build tableInfo + var ( + tbInfo *model.TableInfo + ) + if s.ReferTable != nil { + tbInfo, err = ddl.BuildTableInfoWithLike(ctx, ident, referTbl, s) + } else { + tbInfo, err = ddl.BuildTableInfoWithStmt(ctx, s, schema.Charset, schema.Collate, nil) + } + if err != nil { + return errors.Trace(err) + } + + // TODO: to reuse the constant fold of expression in partition range definition we use CheckTableInfoValidWithStmt, + // but it may also introduce unwanted limit check in DM's use case. Should check it later. + if err = ddl.CheckTableInfoValidWithStmt(ctx, tbInfo, s); err != nil { + return err + } + + onExist := ddl.OnExistError + if s.IfNotExists { + onExist = ddl.OnExistIgnore + } + + return d.CreateTableWithInfo(ctx, schema.Name, tbInfo, onExist) } // CreateTableWithInfo implements the DDL interface. @@ -166,18 +217,83 @@ func (d SchemaTracker) CreateTableWithInfo( info *model.TableInfo, onExist ddl.OnExist, ) error { - panic("not implemented") + schema := d.SchemaByName(dbName) + if schema == nil { + return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(dbName) + } + + oldTable, _ := d.TableByName(dbName, info.Name) + if oldTable != nil { + switch onExist { + case ddl.OnExistIgnore: + return nil + case ddl.OnExistReplace: + return d.PutTable(dbName, info) + default: + return infoschema.ErrTableExists.GenWithStackByArgs(ast.Ident{Schema: dbName, Name: info.Name}) + } + } + return d.PutTable(dbName, info) } // CreateView implements the DDL interface. func (d SchemaTracker) CreateView(ctx sessionctx.Context, s *ast.CreateViewStmt) error { - panic("not implemented") + viewInfo, err := ddl.BuildViewInfo(ctx, s) + if err != nil { + return err + } + + cols := make([]*table.Column, len(s.Cols)) + for i, v := range s.Cols { + cols[i] = table.ToColumn(&model.ColumnInfo{ + Name: v, + ID: int64(i), + Offset: i, + State: model.StatePublic, + }) + } + + tbInfo, err := ddl.BuildTableInfo(ctx, s.ViewName.Name, cols, nil, "", "") + if err != nil { + return err + } + tbInfo.View = viewInfo + + onExist := ddl.OnExistError + if s.OrReplace { + onExist = ddl.OnExistReplace + } + + return d.CreateTableWithInfo(ctx, s.ViewName.Schema, tbInfo, onExist) } // DropTable implements the DDL interface. func (d SchemaTracker) DropTable(ctx sessionctx.Context, stmt *ast.DropTableStmt) (err error) { - panic("not implemented") + notExistTables := make([]string, 0, len(stmt.Tables)) + for _, name := range stmt.Tables { + tb, err := d.TableByName(name.Schema, name.Name) + if err != nil || !tb.IsBaseTable() { + if stmt.IfExists { + continue + } + + id := ast.Ident{Schema: name.Schema, Name: name.Name} + notExistTables = append(notExistTables, id.String()) + // For statement dropping multiple tables, we should return error after try to drop all tables. + continue + } + + // Without IF EXISTS, the statement drops all named tables that do exist, and returns an error indicating which + // nonexisting tables it was unable to drop. + // https://dev.mysql.com/doc/refman/5.7/en/drop-table.html + _ = d.DeleteTable(name.Schema, name.Name) + } + + if len(notExistTables) > 0 { + return infoschema.ErrTableDropExists.GenWithStackByArgs(strings.Join(notExistTables, ",")) + } + return nil } // RecoverTable implements the DDL interface, which is no-op in DM's case. @@ -187,7 +303,31 @@ func (d SchemaTracker) RecoverTable(ctx sessionctx.Context, recoverInfo *ddl.Rec // DropView implements the DDL interface. func (d SchemaTracker) DropView(ctx sessionctx.Context, stmt *ast.DropTableStmt) (err error) { - panic("not implemented") + notExistTables := make([]string, 0, len(stmt.Tables)) + for _, name := range stmt.Tables { + tb, err := d.TableByName(name.Schema, name.Name) + if err != nil { + if stmt.IfExists { + continue + } + + id := ast.Ident{Schema: name.Schema, Name: name.Name} + notExistTables = append(notExistTables, id.String()) + continue + } + + // the behaviour is fast fail when type is wrong. + if !tb.IsView() { + return dbterror.ErrWrongObject.GenWithStackByArgs(name.Schema, name.Name, "VIEW") + } + + _ = d.DeleteTable(name.Schema, name.Name) + } + + if len(notExistTables) > 0 { + return infoschema.ErrTableDropExists.GenWithStackByArgs(strings.Join(notExistTables, ",")) + } + return nil } // CreateIndex implements the DDL interface. @@ -212,7 +352,42 @@ func (d SchemaTracker) TruncateTable(ctx sessionctx.Context, tableIdent ast.Iden // RenameTable implements the DDL interface. func (d SchemaTracker) RenameTable(ctx sessionctx.Context, stmt *ast.RenameTableStmt) error { - panic("not implemented") + oldIdents := make([]ast.Ident, 0, len(stmt.TableToTables)) + newIdents := make([]ast.Ident, 0, len(stmt.TableToTables)) + for _, tablePair := range stmt.TableToTables { + oldIdent := ast.Ident{Schema: tablePair.OldTable.Schema, Name: tablePair.OldTable.Name} + newIdent := ast.Ident{Schema: tablePair.NewTable.Schema, Name: tablePair.NewTable.Name} + oldIdents = append(oldIdents, oldIdent) + newIdents = append(newIdents, newIdent) + } + return d.renameTable(ctx, oldIdents, newIdents, false) +} + +// renameTable is used by RenameTable and AlterTable. +func (d SchemaTracker) renameTable(ctx sessionctx.Context, oldIdents, newIdents []ast.Ident, isAlterTable bool) error { + tablesCache := make(map[string]int64) + is := InfoStoreAdaptor{inner: d.InfoStore} + for i := range oldIdents { + _, _, err := ddl.ExtractTblInfos(is, oldIdents[i], newIdents[i], isAlterTable, tablesCache) + if err != nil { + return err + } + } + + for i := range oldIdents { + tableInfo, err := d.TableByName(oldIdents[i].Schema, oldIdents[i].Name) + if err != nil { + return err + } + if err = d.DeleteTable(oldIdents[i].Schema, oldIdents[i].Name); err != nil { + return err + } + tableInfo.Name = newIdents[i].Name + if err = d.PutTable(newIdents[i].Schema, tableInfo); err != nil { + return err + } + } + return nil } // LockTables implements the DDL interface, it's no-op in DM's case. diff --git a/ddl/schematracker/dm_tracker_test.go b/ddl/schematracker/dm_tracker_test.go new file mode 100644 index 0000000000000..41b9f61c5a8c4 --- /dev/null +++ b/ddl/schematracker/dm_tracker_test.go @@ -0,0 +1,82 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copyright 2013 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSES/QL-LICENSE file. + +package schematracker + +import ( + "fmt" + "testing" + + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/util/mock" + "github.com/stretchr/testify/require" +) + +func TestCreateTableNoNumLimit(t *testing.T) { + sql := "create table test.t_too_large (" + cnt := 3000 + for i := 1; i <= cnt; i++ { + sql += fmt.Sprintf("a%d double, b%d double, c%d double, d%d double", i, i, i, i) + if i != cnt { + sql += "," + } + } + sql += ");" + + sctx := mock.NewContext() + p := parser.New() + stmt, err := p.ParseOneStmt(sql, "", "") + require.NoError(t, err) + + tracker := NewSchemaTracker(2) + tracker.createTestDB() + err = tracker.CreateTable(sctx, stmt.(*ast.CreateTableStmt)) + require.NoError(t, err) + + sql = "create table test.t_too_many_indexes (" + for i := 0; i < 100; i++ { + if i != 0 { + sql += "," + } + sql += fmt.Sprintf("c%d int", i) + } + for i := 0; i < 100; i++ { + sql += "," + sql += fmt.Sprintf("key k%d(c%d)", i, i) + } + sql += ");" + stmt, err = p.ParseOneStmt(sql, "", "") + require.NoError(t, err) + err = tracker.CreateTable(sctx, stmt.(*ast.CreateTableStmt)) + require.NoError(t, err) +} + +func TestCreateTableLongIndex(t *testing.T) { + sql := "create table test.t (c1 int, c2 blob, c3 varchar(64), index idx_c2(c2(555555)));" + + sctx := mock.NewContext() + p := parser.New() + stmt, err := p.ParseOneStmt(sql, "", "") + require.NoError(t, err) + + tracker := NewSchemaTracker(2) + tracker.createTestDB() + err = tracker.CreateTable(sctx, stmt.(*ast.CreateTableStmt)) + require.NoError(t, err) +} diff --git a/executor/ddl_test.go b/executor/ddl_test.go index d3740f5781d57..24da7f4397869 100644 --- a/executor/ddl_test.go +++ b/executor/ddl_test.go @@ -98,8 +98,13 @@ func TestInTxnExecDDLInvalid(t *testing.T) { } func TestCreateTable(t *testing.T) { - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("use test") // Test create an exist database @@ -1494,8 +1499,13 @@ func TestRenameTable(t *testing.T) { defer func() { require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange")) }() - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("drop database if exists rename1") @@ -1577,8 +1587,13 @@ func TestRenameMultiTables(t *testing.T) { defer func() { require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange")) }() - store, clean := testkit.CreateMockStore(t) + store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() + + ddlChecker := schematracker.NewChecker(dom.DDL()) + dom.SetDDL(ddlChecker) + ddlChecker.CreateTestDB() + tk := testkit.NewTestKit(t, store) tk.MustExec("drop database if exists rename1") From 523da29b1013bd23bb9d2ebc98001fbac3913ffd Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Wed, 13 Jul 2022 15:37:05 +0800 Subject: [PATCH 11/27] metrics/grafana: bring back the plan cache miss panel (#36081) close pingcap/tidb#36079 --- metrics/grafana/tidb.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metrics/grafana/tidb.json b/metrics/grafana/tidb.json index e03f651459408..7b15d3d6bbf64 100644 --- a/metrics/grafana/tidb.json +++ b/metrics/grafana/tidb.json @@ -6904,7 +6904,7 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, - "description": "TiDB plan cache hit total", + "description": "TiDB plan cache miss total", "editable": true, "error": false, "fieldConfig": { @@ -6955,7 +6955,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tidb_server_plan_cache_miss_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -6967,7 +6967,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Queries Using Plan Cache OPS", + "title": "Plan Cache Miss OPS", "tooltip": { "msResolution": false, "shared": true, @@ -16119,4 +16119,4 @@ "title": "Test-Cluster-TiDB", "uid": "000000011", "version": 1 -} \ No newline at end of file +} From a0e4ba9dc6119b1acac1c4496cf05c14484d3505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Wed, 13 Jul 2022 09:57:06 +0200 Subject: [PATCH 12/27] planner: Reduce verbosity of logging unknown system variables (#36013) close pingcap/tidb#36011 --- planner/core/expression_rewriter.go | 2 +- session/session.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index d662020cd50e4..67f97245075bb 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -1292,7 +1292,7 @@ func (er *expressionRewriter) rewriteVariable(v *ast.VariableExpr) { } sysVar := variable.GetSysVar(name) if sysVar == nil { - er.err = variable.ErrUnknownSystemVar.GenWithStackByArgs(name) + er.err = variable.ErrUnknownSystemVar.FastGenByArgs(name) if err := variable.CheckSysVarIsRemoved(name); err != nil { // Removed vars still return an error, but we customize it from // "unknown" to an explanation of why it is not supported. diff --git a/session/session.go b/session/session.go index bf5a7a9277caf..8bbf90a7c8550 100644 --- a/session/session.go +++ b/session/session.go @@ -1958,7 +1958,10 @@ func (s *session) ExecuteStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlex // Only print log message when this SQL is from the user. // Mute the warning for internal SQLs. if !s.sessionVars.InRestrictedSQL { - logutil.Logger(ctx).Warn("compile SQL failed", zap.Error(err), zap.String("SQL", stmtNode.Text())) + if !variable.ErrUnknownSystemVar.Equal(err) { + logutil.Logger(ctx).Warn("compile SQL failed", zap.Error(err), + zap.String("SQL", stmtNode.Text())) + } } return nil, err } From f581ec3a45cea9264c2d8a5b90c2acf59f8a78b9 Mon Sep 17 00:00:00 2001 From: xiongjiwei Date: Wed, 13 Jul 2022 16:41:05 +0800 Subject: [PATCH 13/27] test: remove meaningless test and update bazel (#36136) --- br/cmd/br/BUILD.bazel | 1 + br/pkg/conn/BUILD.bazel | 3 +- br/pkg/lightning/restore/BUILD.bazel | 4 +- br/pkg/logutil/BUILD.bazel | 1 + br/pkg/restore/BUILD.bazel | 6 + br/pkg/storage/BUILD.bazel | 1 + br/pkg/stream/BUILD.bazel | 25 +- br/pkg/streamhelper/BUILD.bazel | 81 ++++++ br/pkg/streamhelper/config/BUILD.bazel | 9 + br/pkg/task/BUILD.bazel | 3 + br/pkg/utils/BUILD.bazel | 6 + config/BUILD.bazel | 1 + ddl/BUILD.bazel | 2 + ddl/ddl_test.go | 314 --------------------- ddl/schematracker/BUILD.bazel | 51 ++++ domain/BUILD.bazel | 1 + executor/BUILD.bazel | 1 + metrics/BUILD.bazel | 1 + planner/core/BUILD.bazel | 4 + sessiontxn/BUILD.bazel | 11 +- sessiontxn/internal/BUILD.bazel | 17 ++ sessiontxn/isolation/BUILD.bazel | 2 + sessiontxn/staleread/BUILD.bazel | 1 + tests/realtikvtest/sessiontest/BUILD.bazel | 3 - 24 files changed, 203 insertions(+), 346 deletions(-) create mode 100644 br/pkg/streamhelper/BUILD.bazel create mode 100644 br/pkg/streamhelper/config/BUILD.bazel create mode 100644 ddl/schematracker/BUILD.bazel create mode 100644 sessiontxn/internal/BUILD.bazel diff --git a/br/cmd/br/BUILD.bazel b/br/cmd/br/BUILD.bazel index 2958366d93c4f..278044d0e6db5 100644 --- a/br/cmd/br/BUILD.bazel +++ b/br/cmd/br/BUILD.bazel @@ -22,6 +22,7 @@ go_library( "//br/pkg/redact", "//br/pkg/restore", "//br/pkg/rtree", + "//br/pkg/streamhelper/config", "//br/pkg/summary", "//br/pkg/task", "//br/pkg/trace", diff --git a/br/pkg/conn/BUILD.bazel b/br/pkg/conn/BUILD.bazel index b61518eb2f44d..ad1bcacaaac8d 100644 --- a/br/pkg/conn/BUILD.bazel +++ b/br/pkg/conn/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_kvproto//pkg/brpb", + "@com_github_pingcap_kvproto//pkg/logbackuppb", "@com_github_pingcap_kvproto//pkg/metapb", "@com_github_pingcap_log//:log", "@com_github_tikv_client_go_v2//oracle", @@ -26,9 +27,7 @@ go_library( "@com_github_tikv_client_go_v2//txnkv/txnlock", "@com_github_tikv_pd_client//:client", "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//backoff", "@org_golang_google_grpc//codes", - "@org_golang_google_grpc//credentials", "@org_golang_google_grpc//keepalive", "@org_golang_google_grpc//status", "@org_uber_go_zap//:zap", diff --git a/br/pkg/lightning/restore/BUILD.bazel b/br/pkg/lightning/restore/BUILD.bazel index 80befee3774fd..3586907693301 100644 --- a/br/pkg/lightning/restore/BUILD.bazel +++ b/br/pkg/lightning/restore/BUILD.bazel @@ -38,7 +38,6 @@ go_library( "//br/pkg/utils", "//br/pkg/version", "//br/pkg/version/build", - "//config", "//kv", "//meta/autoid", "//parser", @@ -109,10 +108,13 @@ go_test( "//ddl", "//errno", "//kv", + "//meta", + "//meta/autoid", "//parser", "//parser/ast", "//parser/model", "//parser/mysql", + "//store/mockstore", "//store/pdtypes", "//table/tables", "//types", diff --git a/br/pkg/logutil/BUILD.bazel b/br/pkg/logutil/BUILD.bazel index 5a8df97911de6..1443a6bfd4639 100644 --- a/br/pkg/logutil/BUILD.bazel +++ b/br/pkg/logutil/BUILD.bazel @@ -12,6 +12,7 @@ go_library( deps = [ "//br/pkg/lightning/metric", "//br/pkg/redact", + "//kv", "@com_github_google_uuid//:uuid", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/brpb", diff --git a/br/pkg/restore/BUILD.bazel b/br/pkg/restore/BUILD.bazel index e18abc5e82b59..4f242d49de7e5 100644 --- a/br/pkg/restore/BUILD.bazel +++ b/br/pkg/restore/BUILD.bazel @@ -21,6 +21,7 @@ go_library( importpath = "github.com/pingcap/tidb/br/pkg/restore", visibility = ["//visibility:public"], deps = [ + "//br/pkg/backup", "//br/pkg/checksum", "//br/pkg/conn", "//br/pkg/errors", @@ -45,6 +46,7 @@ go_library( "//statistics/handle", "//store/pdtypes", "//tablecodec", + "//util", "//util/codec", "//util/hack", "//util/mathutil", @@ -109,7 +111,9 @@ go_test( "//br/pkg/mock", "//br/pkg/rtree", "//br/pkg/storage", + "//br/pkg/stream", "//br/pkg/utils", + "//infoschema", "//kv", "//meta/autoid", "//parser/model", @@ -122,8 +126,10 @@ go_test( "//testkit/testsetup", "//types", "//util/codec", + "//util/mathutil", "@com_github_golang_protobuf//proto", "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_kvproto//pkg/brpb", "@com_github_pingcap_kvproto//pkg/encryptionpb", "@com_github_pingcap_kvproto//pkg/errorpb", diff --git a/br/pkg/storage/BUILD.bazel b/br/pkg/storage/BUILD.bazel index 46150497b872b..762df1ae59957 100644 --- a/br/pkg/storage/BUILD.bazel +++ b/br/pkg/storage/BUILD.bazel @@ -33,6 +33,7 @@ go_library( "@com_github_aws_aws_sdk_go//aws/session", "@com_github_aws_aws_sdk_go//service/s3", "@com_github_aws_aws_sdk_go//service/s3/s3iface", + "@com_github_aws_aws_sdk_go//service/s3/s3manager", "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//:azblob", "@com_github_google_uuid//:uuid", diff --git a/br/pkg/stream/BUILD.bazel b/br/pkg/stream/BUILD.bazel index 7d2ac25a863c9..15ee92d85b2a2 100644 --- a/br/pkg/stream/BUILD.bazel +++ b/br/pkg/stream/BUILD.bazel @@ -3,11 +3,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "stream", srcs = [ - "client.go", "decode_kv.go", "meta_kv.go", - "models.go", - "prefix_scanner.go", "rewrite_meta_rawkv.go", "stream_mgr.go", "stream_status.go", @@ -15,14 +12,13 @@ go_library( importpath = "github.com/pingcap/tidb/br/pkg/stream", visibility = ["//visibility:public"], deps = [ - "//br/pkg/backup", - "//br/pkg/conn", "//br/pkg/errors", "//br/pkg/glue", "//br/pkg/httputil", "//br/pkg/logutil", - "//br/pkg/redact", "//br/pkg/storage", + "//br/pkg/streamhelper", + "//br/pkg/utils", "//kv", "//meta", "//parser/model", @@ -31,14 +27,12 @@ go_library( "//util/codec", "//util/table-filter", "@com_github_fatih_color//:color", - "@com_github_gogo_protobuf//proto", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/brpb", "@com_github_pingcap_kvproto//pkg/metapb", "@com_github_pingcap_log//:log", - "@com_github_tikv_client_go_v2//kv", "@com_github_tikv_client_go_v2//oracle", - "@io_etcd_go_etcd_client_v3//:client", + "@com_github_tikv_pd_client//:client", "@org_golang_x_sync//errgroup", "@org_uber_go_zap//:zap", ], @@ -48,28 +42,19 @@ go_test( name = "stream_test", srcs = [ "decode_kv_test.go", - "integration_test.go", "meta_kv_test.go", "rewrite_meta_rawkv_test.go", "stream_misc_test.go", ], embed = [":stream"], deps = [ - "//br/pkg/errors", - "//br/pkg/logutil", - "//br/pkg/storage", + "//br/pkg/streamhelper", "//meta", "//parser/model", "//tablecodec", "//util/codec", - "@com_github_pingcap_errors//:errors", + "//util/table-filter", "@com_github_pingcap_kvproto//pkg/brpb", - "@com_github_pingcap_log//:log", "@com_github_stretchr_testify//require", - "@com_github_tikv_client_go_v2//kv", - "@io_etcd_go_etcd_client_v3//:client", - "@io_etcd_go_etcd_server_v3//embed", - "@io_etcd_go_etcd_server_v3//mvcc", - "@org_uber_go_zap//:zap", ], ) diff --git a/br/pkg/streamhelper/BUILD.bazel b/br/pkg/streamhelper/BUILD.bazel new file mode 100644 index 0000000000000..4af832cbf1c26 --- /dev/null +++ b/br/pkg/streamhelper/BUILD.bazel @@ -0,0 +1,81 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "streamhelper", + srcs = [ + "advancer.go", + "advancer_daemon.go", + "advancer_env.go", + "client.go", + "collector.go", + "models.go", + "prefix_scanner.go", + "regioniter.go", + "stream_listener.go", + "tsheap.go", + ], + importpath = "github.com/pingcap/tidb/br/pkg/streamhelper", + visibility = ["//visibility:public"], + deps = [ + "//br/pkg/errors", + "//br/pkg/logutil", + "//br/pkg/redact", + "//br/pkg/streamhelper/config", + "//br/pkg/utils", + "//config", + "//kv", + "//metrics", + "//owner", + "@com_github_gogo_protobuf//proto", + "@com_github_golang_protobuf//proto", + "@com_github_google_btree//:btree", + "@com_github_google_uuid//:uuid", + "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_kvproto//pkg/brpb", + "@com_github_pingcap_kvproto//pkg/logbackuppb", + "@com_github_pingcap_kvproto//pkg/metapb", + "@com_github_pingcap_log//:log", + "@com_github_tikv_client_go_v2//kv", + "@com_github_tikv_client_go_v2//oracle", + "@com_github_tikv_pd_client//:client", + "@io_etcd_go_etcd_client_v3//:client", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//keepalive", + "@org_golang_x_sync//errgroup", + "@org_uber_go_zap//:zap", + ], +) + +go_test( + name = "streamhelper_test", + srcs = [ + "advancer_test.go", + "basic_lib_for_test.go", + "integration_test.go", + "tsheap_test.go", + ], + deps = [ + ":streamhelper", + "//br/pkg/errors", + "//br/pkg/logutil", + "//br/pkg/storage", + "//br/pkg/streamhelper/config", + "//kv", + "//tablecodec", + "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_kvproto//pkg/brpb", + "@com_github_pingcap_kvproto//pkg/errorpb", + "@com_github_pingcap_kvproto//pkg/logbackuppb", + "@com_github_pingcap_kvproto//pkg/metapb", + "@com_github_pingcap_log//:log", + "@com_github_stretchr_testify//require", + "@com_github_tikv_client_go_v2//kv", + "@io_etcd_go_etcd_client_v3//:client", + "@io_etcd_go_etcd_server_v3//embed", + "@io_etcd_go_etcd_server_v3//mvcc", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + "@org_uber_go_zap//zapcore", + ], +) diff --git a/br/pkg/streamhelper/config/BUILD.bazel b/br/pkg/streamhelper/config/BUILD.bazel new file mode 100644 index 0000000000000..1911b1ac82605 --- /dev/null +++ b/br/pkg/streamhelper/config/BUILD.bazel @@ -0,0 +1,9 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "config", + srcs = ["advancer_conf.go"], + importpath = "github.com/pingcap/tidb/br/pkg/streamhelper/config", + visibility = ["//visibility:public"], + deps = ["@com_github_spf13_pflag//:pflag"], +) diff --git a/br/pkg/task/BUILD.bazel b/br/pkg/task/BUILD.bazel index 9cdc23114f2be..b01d47de6de22 100644 --- a/br/pkg/task/BUILD.bazel +++ b/br/pkg/task/BUILD.bazel @@ -26,6 +26,8 @@ go_library( "//br/pkg/rtree", "//br/pkg/storage", "//br/pkg/stream", + "//br/pkg/streamhelper", + "//br/pkg/streamhelper/config", "//br/pkg/summary", "//br/pkg/utils", "//br/pkg/version", @@ -79,6 +81,7 @@ go_test( "//br/pkg/pdutil", "//br/pkg/restore", "//br/pkg/storage", + "//br/pkg/stream", "//br/pkg/utils", "//config", "//parser/model", diff --git a/br/pkg/utils/BUILD.bazel b/br/pkg/utils/BUILD.bazel index a000479c29696..b708ec2fa7979 100644 --- a/br/pkg/utils/BUILD.bazel +++ b/br/pkg/utils/BUILD.bazel @@ -17,12 +17,14 @@ go_library( "retry.go", "safe_point.go", "schema.go", + "store_manager.go", "worker.go", ], importpath = "github.com/pingcap/tidb/br/pkg/utils", visibility = ["//visibility:public"], deps = [ "//br/pkg/errors", + "//br/pkg/logutil", "//br/pkg/metautil", "//errno", "//parser/model", @@ -38,7 +40,11 @@ go_library( "@com_github_pingcap_log//:log", "@com_github_tikv_client_go_v2//oracle", "@com_github_tikv_pd_client//:client", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//backoff", "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//credentials", + "@org_golang_google_grpc//keepalive", "@org_golang_google_grpc//status", "@org_golang_x_net//http/httpproxy", "@org_golang_x_sync//errgroup", diff --git a/config/BUILD.bazel b/config/BUILD.bazel index 222adfd336194..fd000f17f4b8b 100644 --- a/config/BUILD.bazel +++ b/config/BUILD.bazel @@ -10,6 +10,7 @@ go_library( importpath = "github.com/pingcap/tidb/config", visibility = ["//visibility:public"], deps = [ + "//br/pkg/streamhelper/config", "//parser/terror", "//types/json", "//util/logutil", diff --git a/ddl/BUILD.bazel b/ddl/BUILD.bazel index 3327bf550acb9..f02b31b630067 100644 --- a/ddl/BUILD.bazel +++ b/ddl/BUILD.bazel @@ -172,6 +172,7 @@ go_test( deps = [ "//config", "//ddl/placement", + "//ddl/schematracker", "//ddl/testutil", "//ddl/util", "//domain", @@ -223,6 +224,7 @@ go_test( "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_kvproto//pkg/metapb", "@com_github_pingcap_log//:log", + "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@com_github_tikv_client_go_v2//oracle", "@com_github_tikv_client_go_v2//testutils", diff --git a/ddl/ddl_test.go b/ddl/ddl_test.go index 160d238b51eb5..5a086511fc186 100644 --- a/ddl/ddl_test.go +++ b/ddl/ddl_test.go @@ -16,7 +16,6 @@ package ddl import ( "context" - "fmt" "testing" "time" @@ -33,8 +32,6 @@ import ( "github.com/pingcap/tidb/sessiontxn" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/table" - "github.com/pingcap/tidb/table/tables" - "github.com/pingcap/tidb/testkit/testutil" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/dbterror" "github.com/pingcap/tidb/util/mock" @@ -56,11 +53,6 @@ func (d *ddl) SetInterceptor(i Interceptor) { d.mu.interceptor = i } -// generalWorker returns the general worker. -func (d *ddl) generalWorker() *worker { - return d.workers[generalWorker] -} - // JobNeedGCForTest is only used for test. var JobNeedGCForTest = jobNeedGC @@ -509,299 +501,6 @@ func testCheckSchemaState(test *testing.T, d *ddl, dbInfo *model.DBInfo, state m } } -type testCtxKeyType int - -func (k testCtxKeyType) String() string { - return "test_ctx_key" -} - -const testCtxKey testCtxKeyType = 0 - -func TestReorg(t *testing.T) { - tests := []struct { - isCommonHandle bool - handle kv.Handle - startKey kv.Handle - endKey kv.Handle - }{ - { - false, - kv.IntHandle(100), - kv.IntHandle(1), - kv.IntHandle(0), - }, - { - true, - testutil.MustNewCommonHandle(t, "a", 100, "string"), - testutil.MustNewCommonHandle(t, 100, "string"), - testutil.MustNewCommonHandle(t, 101, "string"), - }, - } - - for _, test := range tests { - t.Run(fmt.Sprintf("isCommandHandle(%v)", test.isCommonHandle), func(t *testing.T) { - store := createMockStore(t) - defer func() { - require.NoError(t, store.Close()) - }() - - d, err := testNewDDLAndStart( - context.Background(), - WithStore(store), - WithLease(testLease), - ) - require.NoError(t, err) - defer func() { - err := d.Stop() - require.NoError(t, err) - }() - - time.Sleep(testLease) - - sctx := testNewContext(d) - - sctx.SetValue(testCtxKey, 1) - require.Equal(t, sctx.Value(testCtxKey), 1) - sctx.ClearValue(testCtxKey) - - err = sessiontxn.NewTxn(context.Background(), sctx) - require.NoError(t, err) - txn, err := sctx.Txn(true) - require.NoError(t, err) - err = txn.Set([]byte("a"), []byte("b")) - require.NoError(t, err) - err = txn.Rollback() - require.NoError(t, err) - - err = sessiontxn.NewTxn(context.Background(), sctx) - require.NoError(t, err) - txn, err = sctx.Txn(true) - require.NoError(t, err) - err = txn.Set([]byte("a"), []byte("b")) - require.NoError(t, err) - err = txn.Commit(context.Background()) - require.NoError(t, err) - - rowCount := int64(10) - handle := test.handle - job := &model.Job{ - ID: 1, - SnapshotVer: 1, // Make sure it is not zero. So the reorgInfo's first is false. - } - err = sessiontxn.NewTxn(context.Background(), sctx) - require.NoError(t, err) - txn, err = sctx.Txn(true) - require.NoError(t, err) - m := meta.NewMeta(txn) - e := &meta.Element{ID: 333, TypeKey: meta.IndexElementKey} - rInfo := &reorgInfo{ - Job: job, - currElement: e, - d: d.ddlCtx, - } - f := func() error { - d.getReorgCtx(job).setRowCount(rowCount) - d.getReorgCtx(job).setNextKey(handle.Encoded()) - time.Sleep(1*ReorgWaitTimeout + 100*time.Millisecond) - return nil - } - mockTbl := tables.MockTableFromMeta(&model.TableInfo{IsCommonHandle: test.isCommonHandle, CommonHandleVersion: 1}) - err = d.generalWorker().runReorgJob(newReorgHandler(m), rInfo, mockTbl.Meta(), d.lease, f) - require.Error(t, err) - - // The longest to wait for 5 seconds to make sure the function of f is returned. - for i := 0; i < 1000; i++ { - time.Sleep(5 * time.Millisecond) - err = d.generalWorker().runReorgJob(newReorgHandler(m), rInfo, mockTbl.Meta(), d.lease, f) - if err == nil { - require.Equal(t, job.RowCount, rowCount) - - // Test whether reorgInfo's Handle is update. - err = txn.Commit(context.Background()) - require.NoError(t, err) - err = sessiontxn.NewTxn(context.Background(), sctx) - require.NoError(t, err) - - m = meta.NewMeta(txn) - info, err1 := getReorgInfo(NewJobContext(), d.ddlCtx, newReorgHandler(m), job, mockTbl, nil) - require.NoError(t, err1) - require.Equal(t, info.StartKey, kv.Key(handle.Encoded())) - require.Equal(t, info.currElement, e) - break - } - } - require.NoError(t, err) - - job = &model.Job{ - ID: 2, - SchemaID: 1, - Type: model.ActionCreateSchema, - Args: []interface{}{model.NewCIStr("test")}, - SnapshotVer: 1, // Make sure it is not zero. So the reorgInfo's first is false. - } - - element := &meta.Element{ID: 123, TypeKey: meta.ColumnElementKey} - info := &reorgInfo{ - Job: job, - d: d.ddlCtx, - currElement: element, - StartKey: test.startKey.Encoded(), - EndKey: test.endKey.Encoded(), - PhysicalTableID: 456, - } - ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) - err = kv.RunInNewTxn(ctx, d.store, false, func(ctx context.Context, txn kv.Transaction) error { - m := meta.NewMeta(txn) - var err1 error - _, err1 = getReorgInfo(NewJobContext(), d.ddlCtx, newReorgHandler(m), job, mockTbl, []*meta.Element{element}) - require.True(t, meta.ErrDDLReorgElementNotExist.Equal(err1)) - require.Equal(t, job.SnapshotVer, uint64(0)) - return nil - }) - require.NoError(t, err) - job.SnapshotVer = uint64(1) - err = info.UpdateReorgMeta(info.StartKey) - require.NoError(t, err) - err = kv.RunInNewTxn(ctx, d.store, false, func(ctx context.Context, txn kv.Transaction) error { - m := meta.NewMeta(txn) - info1, err1 := getReorgInfo(NewJobContext(), d.ddlCtx, newReorgHandler(m), job, mockTbl, []*meta.Element{element}) - require.NoError(t, err1) - require.Equal(t, info1.currElement, info.currElement) - require.Equal(t, info1.StartKey, info.StartKey) - require.Equal(t, info1.EndKey, info.EndKey) - require.Equal(t, info1.PhysicalTableID, info.PhysicalTableID) - return nil - }) - require.NoError(t, err) - - err = d.Stop() - require.NoError(t, err) - err = d.generalWorker().runReorgJob(newReorgHandler(m), rInfo, mockTbl.Meta(), d.lease, func() error { - time.Sleep(4 * testLease) - return nil - }) - require.Error(t, err) - txn, err = sctx.Txn(true) - require.NoError(t, err) - err = txn.Commit(context.Background()) - require.NoError(t, err) - }) - } -} - -func TestCancelJobs(t *testing.T) { - store, clean := newMockStore(t) - defer clean() - - txn, err := store.Begin() - require.NoError(t, err) - - m := meta.NewMeta(txn) - cnt := 10 - ids := make([]int64, cnt) - for i := 0; i < cnt; i++ { - job := &model.Job{ - ID: int64(i), - SchemaID: 1, - Type: model.ActionCreateTable, - } - if i == 0 { - job.State = model.JobStateDone - } - if i == 1 { - job.State = model.JobStateCancelled - } - ids[i] = int64(i) - err = m.EnQueueDDLJob(job) - require.NoError(t, err) - } - - errs, err := CancelJobs(txn, ids) - require.NoError(t, err) - for i, err := range errs { - if i == 0 { - require.Error(t, err) - continue - } - require.NoError(t, err) - } - - errs, err = CancelJobs(txn, []int64{}) - require.NoError(t, err) - require.Nil(t, errs) - - errs, err = CancelJobs(txn, []int64{-1}) - require.NoError(t, err) - require.Error(t, errs[0]) - require.Regexp(t, "DDL Job:-1 not found$", errs[0].Error()) - - // test cancel finish job. - job := &model.Job{ - ID: 100, - SchemaID: 1, - Type: model.ActionCreateTable, - State: model.JobStateDone, - } - err = m.EnQueueDDLJob(job) - require.NoError(t, err) - errs, err = CancelJobs(txn, []int64{100}) - require.NoError(t, err) - require.Error(t, errs[0]) - require.Regexp(t, "This job:100 is finished, so can't be cancelled$", errs[0].Error()) - - // test can't cancelable job. - job.Type = model.ActionDropIndex - job.SchemaState = model.StateWriteOnly - job.State = model.JobStateRunning - job.ID = 101 - err = m.EnQueueDDLJob(job) - require.NoError(t, err) - errs, err = CancelJobs(txn, []int64{101}) - require.NoError(t, err) - require.Error(t, errs[0]) - require.Regexp(t, "This job:101 is almost finished, can't be cancelled now$", errs[0].Error()) - - // When both types of jobs exist in the DDL queue, - // we first cancel the job with a larger ID. - job = &model.Job{ - ID: 1000, - SchemaID: 1, - TableID: 2, - Type: model.ActionAddIndex, - } - job1 := &model.Job{ - ID: 1001, - SchemaID: 1, - TableID: 2, - Type: model.ActionAddColumn, - } - job2 := &model.Job{ - ID: 1002, - SchemaID: 1, - TableID: 2, - Type: model.ActionAddIndex, - } - job3 := &model.Job{ - ID: 1003, - SchemaID: 1, - TableID: 2, - Type: model.ActionRepairTable, - } - require.NoError(t, m.EnQueueDDLJob(job, meta.AddIndexJobListKey)) - require.NoError(t, m.EnQueueDDLJob(job1)) - require.NoError(t, m.EnQueueDDLJob(job2, meta.AddIndexJobListKey)) - require.NoError(t, m.EnQueueDDLJob(job3)) - - errs, err = CancelJobs(txn, []int64{job1.ID, job.ID, job2.ID, job3.ID}) - require.NoError(t, err) - for _, err := range errs { - require.NoError(t, err) - } - - err = txn.Rollback() - require.NoError(t, err) -} - func TestError(t *testing.T) { kvErrs := []*terror.Error{ dbterror.ErrDDLJobNotFound, @@ -814,16 +513,3 @@ func TestError(t *testing.T) { require.Equal(t, uint16(err.Code()), code) } } - -func newMockStore(t *testing.T) (store kv.Storage, clean func()) { - var err error - store, err = mockstore.NewMockStore() - require.NoError(t, err) - - clean = func() { - err = store.Close() - require.NoError(t, err) - } - - return -} diff --git a/ddl/schematracker/BUILD.bazel b/ddl/schematracker/BUILD.bazel new file mode 100644 index 0000000000000..a61470a7a1192 --- /dev/null +++ b/ddl/schematracker/BUILD.bazel @@ -0,0 +1,51 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "schematracker", + srcs = [ + "checker.go", + "dm_tracker.go", + "info_store.go", + ], + importpath = "github.com/pingcap/tidb/ddl/schematracker", + visibility = ["//visibility:public"], + deps = [ + "//ddl", + "//ddl/util", + "//executor", + "//infoschema", + "//kv", + "//meta/autoid", + "//owner", + "//parser/ast", + "//parser/charset", + "//parser/model", + "//sessionctx", + "//sessionctx/variable", + "//statistics/handle", + "//table", + "//table/tables", + "//tidb-binlog/pump_client", + "//util/collate", + "//util/dbterror", + "@com_github_ngaut_pools//:pools", + "@com_github_pingcap_errors//:errors", + ], +) + +go_test( + name = "schematracker_test", + srcs = [ + "dm_tracker_test.go", + "info_store_test.go", + ], + embed = [":schematracker"], + deps = [ + "//infoschema", + "//parser", + "//parser/ast", + "//parser/model", + "//util/mock", + "@com_github_stretchr_testify//require", + ], +) diff --git a/domain/BUILD.bazel b/domain/BUILD.bazel index b1c867746ac8d..d878a32ded8a6 100644 --- a/domain/BUILD.bazel +++ b/domain/BUILD.bazel @@ -17,6 +17,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//bindinfo", + "//br/pkg/streamhelper", "//config", "//ddl", "//ddl/util", diff --git a/executor/BUILD.bazel b/executor/BUILD.bazel index 6b5590f9429a0..46b816e9128cc 100644 --- a/executor/BUILD.bazel +++ b/executor/BUILD.bazel @@ -328,6 +328,7 @@ go_test( "//config", "//ddl", "//ddl/placement", + "//ddl/schematracker", "//ddl/testutil", "//ddl/util", "//distsql", diff --git a/metrics/BUILD.bazel b/metrics/BUILD.bazel index 6ff2486d8c333..ad34ac544fa0e 100644 --- a/metrics/BUILD.bazel +++ b/metrics/BUILD.bazel @@ -9,6 +9,7 @@ go_library( "domain.go", "executor.go", "gc_worker.go", + "log_backup.go", "meta.go", "metrics.go", "owner.go", diff --git a/planner/core/BUILD.bazel b/planner/core/BUILD.bazel index 83342809499ce..e2ebf20dea339 100644 --- a/planner/core/BUILD.bazel +++ b/planner/core/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "core", srcs = [ + "access_object.go", "cache.go", "cacheable_checker.go", "collect_column_stats_usage.go", @@ -13,6 +14,7 @@ go_library( "explain.go", "expression_rewriter.go", "find_best_task.go", + "flat_plan.go", "fragment.go", "handle_cols.go", "hashcode.go", @@ -116,6 +118,7 @@ go_library( "//util/kvcache", "//util/logutil", "//util/mathutil", + "//util/memory", "//util/mock", "//util/paging", "//util/parser", @@ -154,6 +157,7 @@ go_test( "expression_rewriter_test.go", "expression_test.go", "find_best_task_test.go", + "flat_plan_test.go", "fragment_test.go", "indexmerge_test.go", "integration_partition_test.go", diff --git a/sessiontxn/BUILD.bazel b/sessiontxn/BUILD.bazel index 0738636e8299a..c884d7981e19e 100644 --- a/sessiontxn/BUILD.bazel +++ b/sessiontxn/BUILD.bazel @@ -4,8 +4,8 @@ go_library( name = "sessiontxn", srcs = [ "failpoint.go", + "future.go", "interface.go", - "txn.go", ], importpath = "github.com/pingcap/tidb/sessiontxn", visibility = ["//visibility:public"], @@ -14,12 +14,7 @@ go_library( "//kv", "//parser/ast", "//sessionctx", - "//sessionctx/variable", - "//table/temptable", "//util/stringutil", - "@com_github_opentracing_opentracing_go//:opentracing-go", - "@com_github_pingcap_kvproto//pkg/kvrpcpb", - "@com_github_tikv_client_go_v2//oracle", ], ) @@ -35,12 +30,16 @@ go_test( "//infoschema", "//kv", "//parser/ast", + "//parser/model", "//planner/core", "//sessionctx", + "//sessiontxn/internal", "//sessiontxn/staleread", + "//tablecodec", "//testkit", "//testkit/testfork", "//testkit/testsetup", + "//tests/realtikvtest", "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", "@com_github_tikv_client_go_v2//oracle", diff --git a/sessiontxn/internal/BUILD.bazel b/sessiontxn/internal/BUILD.bazel new file mode 100644 index 0000000000000..43e4dc2b8be1c --- /dev/null +++ b/sessiontxn/internal/BUILD.bazel @@ -0,0 +1,17 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "internal", + srcs = ["txn.go"], + importpath = "github.com/pingcap/tidb/sessiontxn/internal", + visibility = ["//sessiontxn:__subpackages__"], + deps = [ + "//kv", + "//sessionctx", + "//sessionctx/variable", + "//table/temptable", + "//util/logutil", + "@com_github_pingcap_kvproto//pkg/kvrpcpb", + "@org_uber_go_zap//:zap", + ], +) diff --git a/sessiontxn/isolation/BUILD.bazel b/sessiontxn/isolation/BUILD.bazel index a05b08583768a..8ef29433ceb7d 100644 --- a/sessiontxn/isolation/BUILD.bazel +++ b/sessiontxn/isolation/BUILD.bazel @@ -22,9 +22,11 @@ go_library( "//sessionctx", "//sessionctx/variable", "//sessiontxn", + "//sessiontxn/internal", "//sessiontxn/staleread", "//table/temptable", "//util/logutil", + "@com_github_opentracing_opentracing_go//:opentracing-go", "@com_github_pingcap_errors//:errors", "@com_github_tikv_client_go_v2//error", "@com_github_tikv_client_go_v2//oracle", diff --git a/sessiontxn/staleread/BUILD.bazel b/sessiontxn/staleread/BUILD.bazel index d6272550153af..117623298d563 100644 --- a/sessiontxn/staleread/BUILD.bazel +++ b/sessiontxn/staleread/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "//sessionctx", "//sessionctx/variable", "//sessiontxn", + "//sessiontxn/internal", "//table/temptable", "//types", "//util/dbterror", diff --git a/tests/realtikvtest/sessiontest/BUILD.bazel b/tests/realtikvtest/sessiontest/BUILD.bazel index 14923fa0fa623..1f7df628a05d9 100644 --- a/tests/realtikvtest/sessiontest/BUILD.bazel +++ b/tests/realtikvtest/sessiontest/BUILD.bazel @@ -17,7 +17,6 @@ go_test( "//domain", "//errno", "//executor", - "//infoschema", "//kv", "//meta/autoid", "//parser", @@ -31,11 +30,9 @@ go_test( "//session", "//sessionctx", "//sessionctx/variable", - "//sessiontxn", "//store/copr", "//store/mockstore", "//table/tables", - "//tablecodec", "//testkit", "//tests/realtikvtest", "//types", From 12c172163bb4e2263adf60ef56db32ecccb7d33b Mon Sep 17 00:00:00 2001 From: tangenta Date: Wed, 13 Jul 2022 17:39:05 +0800 Subject: [PATCH 14/27] ddl: support rename index and columns for multi-schema change (#36148) ref pingcap/tidb#14766 --- ddl/column.go | 13 ++ ddl/index.go | 20 ++- ddl/multi_schema_change.go | 11 ++ ddl/multi_schema_change_test.go | 259 ++++++++++++++++++++++++++++++++ 4 files changed, 301 insertions(+), 2 deletions(-) diff --git a/ddl/column.go b/ddl/column.go index 1ac52bc902242..b31300d07e0ae 100644 --- a/ddl/column.go +++ b/ddl/column.go @@ -1349,6 +1349,12 @@ func (w *worker) doModifyColumn( } } + if job.MultiSchemaInfo != nil && job.MultiSchemaInfo.Revertible { + job.MarkNonRevertible() + // Store the mark and enter the next DDL handling loop. + return updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, false) + } + if err := adjustTableInfoAfterModifyColumn(tblInfo, newCol, oldCol, pos); err != nil { job.State = model.JobStateRollingback return ver, errors.Trace(err) @@ -1505,6 +1511,13 @@ func updateColumnDefaultValue(d *ddlCtx, t *meta.Meta, job *model.Job, newCol *m if err != nil { return ver, errors.Trace(err) } + + if job.MultiSchemaInfo != nil && job.MultiSchemaInfo.Revertible { + job.MarkNonRevertible() + // Store the mark and enter the next DDL handling loop. + return updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, false) + } + oldCol := model.FindColumnInfo(tblInfo.Columns, oldColName.L) if oldCol == nil || oldCol.State != model.StatePublic { job.State = model.JobStateCancelled diff --git a/ddl/index.go b/ddl/index.go index fa9193d0921ec..ac0f74a0e384d 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -323,8 +323,13 @@ func onRenameIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) return ver, errors.Trace(dbterror.ErrOptOnCacheTable.GenWithStackByArgs("Rename Index")) } - idx := tblInfo.FindIndexByName(from.L) - idx.Name = to + if job.MultiSchemaInfo != nil && job.MultiSchemaInfo.Revertible { + job.MarkNonRevertible() + // Store the mark and enter the next DDL handling loop. + return updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, false) + } + + renameIndexes(tblInfo, from, to) if ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true); err != nil { job.State = model.JobStateCancelled return ver, errors.Trace(err) @@ -1596,3 +1601,14 @@ func findIdxCol(idxInfo *model.IndexInfo, colName model.CIStr) int { } return -1 } + +func renameIndexes(tblInfo *model.TableInfo, from, to model.CIStr) { + for _, idx := range tblInfo.Indices { + if idx.Name.L == from.L { + idx.Name = to + } else if isTempIdxInfo(idx, tblInfo) && getChangingIndexOriginName(idx) == from.O { + idx.Name.L = strings.Replace(idx.Name.L, from.L, to.L, 1) + idx.Name.O = strings.Replace(idx.Name.O, from.O, to.O, 1) + } + } +} diff --git a/ddl/multi_schema_change.go b/ddl/multi_schema_change.go index 9b6773d5edf79..a7ce46fc2aae7 100644 --- a/ddl/multi_schema_change.go +++ b/ddl/multi_schema_change.go @@ -27,6 +27,9 @@ import ( ) func (d *ddl) MultiSchemaChange(ctx sessionctx.Context, ti ast.Ident) error { + if len(ctx.GetSessionVars().StmtCtx.MultiSchemaInfo.SubJobs) == 0 { + return nil + } schema, t, err := d.getSchemaAndTableByIdent(ctx, ti) if err != nil { return errors.Trace(err) @@ -223,6 +226,11 @@ func fillMultiSchemaInfo(info *model.MultiSchemaInfo, job *model.Job) (err error } } } + case model.ActionRenameIndex: + from := job.Args[0].(model.CIStr) + to := job.Args[1].(model.CIStr) + info.AddIndexes = append(info.AddIndexes, to) + info.DropIndexes = append(info.DropIndexes, from) case model.ActionModifyColumn: newCol := *job.Args[0].(**model.ColumnInfo) oldColName := job.Args[1].(model.CIStr) @@ -236,6 +244,9 @@ func fillMultiSchemaInfo(info *model.MultiSchemaInfo, job *model.Job) (err error if pos != nil && pos.Tp == ast.ColumnPositionAfter { info.PositionColumns = append(info.PositionColumns, pos.RelativeColumn.Name) } + case model.ActionSetDefaultValue: + col := job.Args[0].(*table.Column) + info.ModifyColumns = append(info.ModifyColumns, col.Name) default: return dbterror.ErrRunMultiSchemaChanges } diff --git a/ddl/multi_schema_change_test.go b/ddl/multi_schema_change_test.go index 9f80cd19fbab4..eeede7a8ea954 100644 --- a/ddl/multi_schema_change_test.go +++ b/ddl/multi_schema_change_test.go @@ -315,6 +315,200 @@ func TestMultiSchemaChangeAddDropColumns(t *testing.T) { tk.MustGetErrCode("alter table t add column c int default 3 after a, add column d int default 4 first, drop column a, drop column b;", errno.ErrUnsupportedDDLOperation) } +func TestMultiSchemaChangeRenameColumns(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + originHook := dom.DDL().GetHook() + defer clean() + + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + // unsupported ddl operations + { + // Test add and rename to same column name + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2);") + tk.MustExec("insert into t values ();") + tk.MustGetErrCode("alter table t rename column b to c, add column c int", errno.ErrUnsupportedDDLOperation) + + // Test add column related with rename column + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2);") + tk.MustExec("insert into t values ();") + tk.MustGetErrCode("alter table t rename column b to c, add column e int after b", errno.ErrUnsupportedDDLOperation) + + // Test drop and rename with same column + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2);") + tk.MustExec("insert into t values ();") + tk.MustGetErrCode("alter table t drop column b, rename column b to c", errno.ErrUnsupportedDDLOperation) + + // Test add index and rename with same column + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2, index t(a, b));") + tk.MustExec("insert into t values ();") + tk.MustGetErrCode("alter table t rename column b to c, add index t1(a, b)", errno.ErrUnsupportedDDLOperation) + } + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2, index t(a, b));") + tk.MustExec("insert into t values ();") + tk.MustExec("alter table t rename column b to c, add column e int default 3") + tk.MustQuery("select c from t").Check(testkit.Rows("2")) + tk.MustQuery("select * from t").Check(testkit.Rows("1 2 3")) + + // Test cancel job with rename columns + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int default 1, b int default 2)") + tk.MustExec("insert into t values ()") + hook := newCancelJobHook(store, dom, func(job *model.Job) bool { + // Cancel job when the column 'c' is in write-reorg. + return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization + }) + dom.DDL().SetHook(hook) + tk.MustGetErrCode("alter table t add column c int default 3, rename column b to d;", errno.ErrCancelledDDLJob) + dom.DDL().SetHook(originHook) + tk.MustQuery("select b from t").Check(testkit.Rows("2")) + tk.MustGetErrCode("select d from t", errno.ErrBadField) + + // Test dml stmts when do rename + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int default 1, b int default 2)") + tk.MustExec("insert into t values ()") + hook1 := &ddl.TestDDLCallback{Do: dom} + hook1.OnJobRunBeforeExported = func(job *model.Job) { + assert.Equal(t, model.ActionMultiSchemaChange, job.Type) + if job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization { + rs, _ := tk.Exec("select b from t") + assert.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][0], "2") + } + } + dom.DDL().SetHook(hook1) + tk.MustExec("alter table t add column c int default 3, rename column b to d;") + dom.DDL().SetHook(originHook) + tk.MustQuery("select d from t").Check(testkit.Rows("2")) + tk.MustGetErrCode("select b from t", errno.ErrBadField) +} + +func TestMultiSchemaChangeAlterColumns(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + originHook := dom.DDL().GetHook() + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + // unsupported ddl operations + { + // Test alter and drop with same column + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2);") + tk.MustExec("insert into t values ();") + tk.MustGetErrCode("alter table t alter column b set default 3, drop column b", errno.ErrUnsupportedDDLOperation) + + // Test alter and rename with same column + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2);") + tk.MustExec("insert into t values ();") + tk.MustGetErrCode("alter table t alter column b set default 3, rename column b to c", errno.ErrUnsupportedDDLOperation) + + // Test alter and drop modify same column + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2);") + tk.MustExec("insert into t values ();") + tk.MustGetErrCode("alter table t alter column b set default 3, modify column b double", errno.ErrUnsupportedDDLOperation) + } + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2, index t(a, b));") + tk.MustExec("insert into t values ();") + tk.MustQuery("select * from t").Check(testkit.Rows("1 2")) + tk.MustExec("alter table t rename column a to c, alter column b set default 3;") + tk.MustExec("truncate table t;") + tk.MustExec("insert into t values ();") + tk.MustQuery("select * from t").Check(testkit.Rows("1 3")) + + // Test cancel job with alter columns + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int default 1, b int default 2)") + hook := newCancelJobHook(store, dom, func(job *model.Job) bool { + // Cancel job when the column 'a' is in write-reorg. + return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization + }) + dom.DDL().SetHook(hook) + tk.MustGetErrCode("alter table t add column c int default 3, alter column b set default 3;", errno.ErrCancelledDDLJob) + dom.DDL().SetHook(originHook) + tk.MustExec("insert into t values ()") + tk.MustQuery("select * from t").Check(testkit.Rows("1 2")) + + // Test dml stmts when do alter + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int default 1, b int default 2)") + hook1 := &ddl.TestDDLCallback{Do: dom} + hook1.OnJobRunBeforeExported = func(job *model.Job) { + assert.Equal(t, model.ActionMultiSchemaChange, job.Type) + if job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteOnly { + tk.Exec("insert into t values ()") + } + } + dom.DDL().SetHook(hook1) + tk.MustExec("alter table t add column c int default 3, alter column b set default 3;") + dom.DDL().SetHook(originHook) + tk.MustQuery("select * from t").Check(testkit.Rows("1 2 3")) +} + +func TestMultiSchemaChangeChangeColumns(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + originHook := dom.DDL().GetHook() + defer clean() + + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + // unsupported ddl operations + { + // Test change and drop with same column + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2);") + tk.MustExec("insert into t values ();") + tk.MustGetErrCode("alter table t change column b c double, drop column b", errno.ErrUnsupportedDDLOperation) + + // Test change and add with same column + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2);") + tk.MustExec("insert into t values ();") + tk.MustGetErrCode("alter table t change column b c double, add column c int", errno.ErrUnsupportedDDLOperation) + + // Test add index and rename with same column + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2, index t(a, b));") + tk.MustExec("insert into t values ();") + tk.MustGetErrCode("alter table t change column b c double, add index t1(a, b)", errno.ErrUnsupportedDDLOperation) + } + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int default 1, b int default 2, index t(a, b));") + tk.MustExec("insert into t values ();") + tk.MustExec("alter table t rename column b to c, change column a e bigint default 3;") + tk.MustQuery("select e,c from t").Check(testkit.Rows("1 2")) + tk.MustExec("truncate table t;") + tk.MustExec("insert into t values ();") + tk.MustQuery("select e,c from t").Check(testkit.Rows("3 2")) + + // Test cancel job with change columns + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int default 1, b int default 2)") + tk.MustExec("insert into t values ()") + hook := newCancelJobHook(store, dom, func(job *model.Job) bool { + // Cancel job when the column 'c' is in write-reorg. + return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization + }) + dom.DDL().SetHook(hook) + tk.MustGetErrCode("alter table t add column c int default 3, change column b d bigint default 4;", errno.ErrCancelledDDLJob) + dom.DDL().SetHook(originHook) + tk.MustQuery("select b from t").Check(testkit.Rows("2")) + tk.MustGetErrCode("select d from t", errno.ErrBadField) +} + func TestMultiSchemaChangeAddIndexes(t *testing.T) { store, clean := testkit.CreateMockStore(t) defer clean() @@ -505,6 +699,55 @@ func TestMultiSchemaChangeAddDropIndexes(t *testing.T) { tk.MustExec("admin check table t;") } +func TestMultiSchemaChangeRenameIndexes(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + originHook := dom.DDL().GetHook() + + // Test rename index. + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int, b int, c int, index t(a), index t1(b))") + tk.MustExec("alter table t rename index t to x, rename index t1 to x1") + tk.MustExec("select * from t use index (x);") + tk.MustExec("select * from t use index (x1);") + tk.MustGetErrCode("select * from t use index (t);", errno.ErrKeyDoesNotExist) + tk.MustGetErrCode("select * from t use index (t1);", errno.ErrKeyDoesNotExist) + + // Test drop and rename same index. + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int, b int, c int, index t(a))") + tk.MustGetErrCode("alter table t drop index t, rename index t to t1", errno.ErrUnsupportedDDLOperation) + + // Test add and rename to same index name. + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int, b int, c int, index t(a))") + tk.MustGetErrCode("alter table t add index t1(b), rename index t to t1", errno.ErrUnsupportedDDLOperation) + + // Test drop column with rename index. + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int default 1, b int default 2, c int default 3, index t(a))") + tk.MustExec("insert into t values ();") + tk.MustExec("alter table t drop column a, rename index t to x") + tk.MustGetErrCode("select * from t use index (x);", errno.ErrKeyDoesNotExist) + tk.MustQuery("select * from t;").Check(testkit.Rows("2 3")) + + // Test cancel job with renameIndex + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int default 1, b int default 2, index t(a))") + tk.MustExec("insert into t values ()") + hook := newCancelJobHook(store, dom, func(job *model.Job) bool { + // Cancel job when the column 'c' is in write-reorg. + return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization + }) + dom.DDL().SetHook(hook) + tk.MustGetErrCode("alter table t add column c int default 3, rename index t to t1;", errno.ErrCancelledDDLJob) + dom.DDL().SetHook(originHook) + tk.MustQuery("select * from t use index (t);").Check(testkit.Rows("1 2")) + tk.MustGetErrCode("select * from t use index (t1);", errno.ErrKeyDoesNotExist) +} + func TestMultiSchemaChangeModifyColumns(t *testing.T) { store, clean := testkit.CreateMockStore(t) defer clean() @@ -591,6 +834,22 @@ func TestMultiSchemaChangeModifyColumns(t *testing.T) { tk.MustQuery("select * from t use index(i1, i2);").Check(testkit.Rows("1 3 2", "11 33 22")) tk.MustExec("admin check table t;") + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a bigint null default '1761233443433596323', index t(a));") + tk.MustExec("insert into t set a = '-7184819032643664798';") + tk.MustGetErrCode("alter table t change column a b datetime null default '8972-12-24 10:56:03', rename index t to t1;", errno.ErrTruncatedWrongValue) + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int, b double, index i(a, b));") + tk.MustExec("alter table t rename index i to i1, change column b c int;") + tk.MustQuery("select count(*) from information_schema.TIDB_INDEXES where TABLE_NAME='t' and COLUMN_NAME='c' and KEY_NAME='i1';").Check(testkit.Rows("1")) + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int, b double, index i(a, b), index _Idx$_i(a, b));") + tk.MustExec("alter table t rename index i to i1, change column b c int;") + tk.MustQuery("select count(*) from information_schema.TIDB_INDEXES where TABLE_NAME='t' and COLUMN_NAME='c' and KEY_NAME='i1';").Check(testkit.Rows("1")) + tk.MustQuery("select count(*) from information_schema.TIDB_INDEXES where TABLE_NAME='t' and COLUMN_NAME='c' and KEY_NAME='_Idx$_i';").Check(testkit.Rows("1")) + tk.MustExec("drop table if exists t;") tk.MustExec("create table t (a int, _Col$_a double, index _Idx$_i(a, _Col$_a), index i(a, _Col$_a));") tk.MustExec("alter table t modify column a tinyint;") From 5e8f09b0e4d5864e60e869907fcf2073c2a25b00 Mon Sep 17 00:00:00 2001 From: xiongjiwei Date: Wed, 13 Jul 2022 18:19:05 +0800 Subject: [PATCH 15/27] test: refactor restart test (#36174) --- ddl/ddl_test.go | 143 ------------------------------ ddl/restart_test.go | 212 ++++++++++++-------------------------------- 2 files changed, 58 insertions(+), 297 deletions(-) diff --git a/ddl/ddl_test.go b/ddl/ddl_test.go index 5a086511fc186..54eb13b955629 100644 --- a/ddl/ddl_test.go +++ b/ddl/ddl_test.go @@ -29,7 +29,6 @@ import ( "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/parser/terror" "github.com/pingcap/tidb/sessionctx" - "github.com/pingcap/tidb/sessiontxn" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" @@ -77,60 +76,6 @@ func createMockStore(t *testing.T) kv.Storage { return store } -func testNewContext(d *ddl) sessionctx.Context { - ctx := mock.NewContext() - ctx.Store = d.store - return ctx -} - -func getSchemaVer(t *testing.T, ctx sessionctx.Context) int64 { - err := sessiontxn.NewTxn(context.Background(), ctx) - require.NoError(t, err) - txn, err := ctx.Txn(true) - require.NoError(t, err) - m := meta.NewMeta(txn) - ver, err := m.GetSchemaVersion() - require.NoError(t, err) - return ver -} - -type historyJobArgs struct { - ver int64 - db *model.DBInfo - tbl *model.TableInfo - tblIDs map[int64]struct{} -} - -func checkEqualTable(t *testing.T, t1, t2 *model.TableInfo) { - require.Equal(t, t1.ID, t2.ID) - require.Equal(t, t1.Name, t2.Name) - require.Equal(t, t1.Charset, t2.Charset) - require.Equal(t, t1.Collate, t2.Collate) - require.Equal(t, t1.PKIsHandle, t2.PKIsHandle) - require.Equal(t, t1.Comment, t2.Comment) - require.Equal(t, t1.AutoIncID, t2.AutoIncID) -} - -func checkHistoryJobArgs(t *testing.T, ctx sessionctx.Context, id int64, args *historyJobArgs) { - historyJob, err := GetHistoryJobByID(ctx, id) - require.NoError(t, err) - require.Greater(t, historyJob.BinlogInfo.FinishedTS, uint64(0)) - - if args.tbl != nil { - require.Equal(t, historyJob.BinlogInfo.SchemaVersion, args.ver) - checkEqualTable(t, historyJob.BinlogInfo.TableInfo, args.tbl) - return - } - - // for handling schema job - require.Equal(t, historyJob.BinlogInfo.SchemaVersion, args.ver) - require.Equal(t, historyJob.BinlogInfo.DBInfo, args.db) - // only for creating schema job - if args.db != nil && len(args.tblIDs) == 0 { - return - } -} - func TestGetIntervalFromPolicy(t *testing.T) { policy := []time.Duration{ 1 * time.Second, @@ -413,94 +358,6 @@ func TestNotifyDDLJob(t *testing.T) { } } -func testSchemaInfo(d *ddl, name string) (*model.DBInfo, error) { - dbInfo := &model.DBInfo{ - Name: model.NewCIStr(name), - } - genIDs, err := d.genGlobalIDs(1) - if err != nil { - return nil, err - } - dbInfo.ID = genIDs[0] - return dbInfo, nil -} - -func testCreateSchema(t *testing.T, ctx sessionctx.Context, d *ddl, dbInfo *model.DBInfo) *model.Job { - job := &model.Job{ - SchemaID: dbInfo.ID, - Type: model.ActionCreateSchema, - BinlogInfo: &model.HistoryInfo{}, - Args: []interface{}{dbInfo}, - } - ctx.SetValue(sessionctx.QueryString, "skip") - require.NoError(t, d.DoDDLJob(ctx, job)) - - v := getSchemaVer(t, ctx) - dbInfo.State = model.StatePublic - checkHistoryJobArgs(t, ctx, job.ID, &historyJobArgs{ver: v, db: dbInfo}) - dbInfo.State = model.StateNone - return job -} - -func buildDropSchemaJob(dbInfo *model.DBInfo) *model.Job { - return &model.Job{ - SchemaID: dbInfo.ID, - Type: model.ActionDropSchema, - BinlogInfo: &model.HistoryInfo{}, - } -} - -func testDropSchema(t *testing.T, ctx sessionctx.Context, d *ddl, dbInfo *model.DBInfo) (*model.Job, int64) { - job := buildDropSchemaJob(dbInfo) - ctx.SetValue(sessionctx.QueryString, "skip") - err := d.DoDDLJob(ctx, job) - require.NoError(t, err) - ver := getSchemaVer(t, ctx) - return job, ver -} - -func isDDLJobDone(test *testing.T, t *meta.Meta) bool { - job, err := t.GetDDLJobByIdx(0) - require.NoError(test, err) - if job == nil { - return true - } - - time.Sleep(testLease) - return false -} - -func testCheckSchemaState(test *testing.T, d *ddl, dbInfo *model.DBInfo, state model.SchemaState) { - isDropped := true - - ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) - for { - err := kv.RunInNewTxn(ctx, d.store, false, func(ctx context.Context, txn kv.Transaction) error { - t := meta.NewMeta(txn) - info, err := t.GetDatabase(dbInfo.ID) - require.NoError(test, err) - - if state == model.StateNone { - isDropped = isDDLJobDone(test, t) - if !isDropped { - return nil - } - require.Nil(test, info) - return nil - } - - require.Equal(test, info.Name, dbInfo.Name) - require.Equal(test, info.State, state) - return nil - }) - require.NoError(test, err) - - if isDropped { - break - } - } -} - func TestError(t *testing.T) { kvErrs := []*terror.Error{ dbterror.ErrDDLJobNotFound, diff --git a/ddl/restart_test.go b/ddl/restart_test.go index bbec12b40ca6f..130e988a9367f 100644 --- a/ddl/restart_test.go +++ b/ddl/restart_test.go @@ -11,75 +11,68 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -//go:build !race -package ddl +package ddl_test import ( "context" "errors" - "fmt" "testing" "time" + "github.com/ngaut/pools" + "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/parser/model" - "github.com/pingcap/tidb/parser/mysql" - "github.com/pingcap/tidb/parser/terror" "github.com/pingcap/tidb/sessionctx" - "github.com/pingcap/tidb/types" - "github.com/pingcap/tidb/util/mock" + "github.com/pingcap/tidb/testkit" "github.com/stretchr/testify/require" ) // this test file include some test that will cause data race, mainly because restartWorkers modify d.ctx -func getDDLSchemaVer(t *testing.T, d *ddl) int64 { +func getDDLSchemaVer(t *testing.T, d ddl.DDL) int64 { m, err := d.Stats(nil) require.NoError(t, err) - v := m[ddlSchemaVersion] + v := m["ddl_schema_version"] return v.(int64) } -// restartWorkers is like the function of d.start. But it won't initialize the "workers" and create a new worker. -// It only starts the original workers. -func (d *ddl) restartWorkers(ctx context.Context) { - d.ctx, d.cancel = context.WithCancel(ctx) - - d.wg.Run(d.limitDDLJobs) - if !RunWorker { - return - } - - err := d.ownerManager.CampaignOwner() - terror.Log(err) - for _, worker := range d.workers { - worker.wg.Add(1) - worker.ctx = d.ctx - w := worker - go w.start(d.ddlCtx) - asyncNotify(worker.ddlJobCh) - } +// restartWorkers will stop the old DDL and create a new DDL and start it. +func restartWorkers(t *testing.T, store kv.Storage, d *domain.Domain) { + err := d.DDL().Stop() + require.NoError(t, err) + newDDL := ddl.NewDDL(context.Background(), ddl.WithStore(d.Store()), ddl.WithInfoCache(d.InfoCache()), ddl.WithLease(d.DDL().GetLease())) + d.SetDDL(newDDL) + err = newDDL.Start(pools.NewResourcePool(func() (pools.Resource, error) { + session := testkit.NewTestKit(t, store).Session() + session.GetSessionVars().CommonGlobalLoaded = true + return session, nil + }, 128, 128, 5)) + require.NoError(t, err) } // runInterruptedJob should be called concurrently with restartWorkers -func runInterruptedJob(d *ddl, job *model.Job, doneCh chan error) { - ctx := mock.NewContext() - ctx.Store = d.store - +func runInterruptedJob(t *testing.T, store kv.Storage, d ddl.DDL, job *model.Job, doneCh chan error) { var ( history *model.Job err error ) + ctx := testkit.NewTestKit(t, store).Session() ctx.SetValue(sessionctx.QueryString, "skip") err = d.DoDDLJob(ctx, job) if errors.Is(err, context.Canceled) { endlessLoopTime := time.Now().Add(time.Minute) for history == nil { // imitate DoDDLJob's logic, quit only find history - history, _ = d.getHistoryDDLJob(job.ID) + err = kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), store, false, func(ctx context.Context, txn kv.Transaction) error { + history, err = meta.NewMeta(txn).GetHistoryDDLJob(job.ID) + return err + }) + require.NoError(t, err) if history != nil { err = history.Error } @@ -94,18 +87,16 @@ func runInterruptedJob(d *ddl, job *model.Job, doneCh chan error) { doneCh <- err } -func testRunInterruptedJob(t *testing.T, d *ddl, job *model.Job) { +func testRunInterruptedJob(t *testing.T, store kv.Storage, d *domain.Domain, job *model.Job) { done := make(chan error, 1) - go runInterruptedJob(d, job, done) + go runInterruptedJob(t, store, d.DDL(), job, done) - ticker := time.NewTicker(d.lease * 1) + ticker := time.NewTicker(d.DDL().GetLease()) defer ticker.Stop() for { select { case <-ticker.C: - err := d.Stop() - require.NoError(t, err) - d.restartWorkers(context.Background()) + restartWorkers(t, store, d) time.Sleep(time.Millisecond * 20) case err := <-done: require.Nil(t, err) @@ -115,24 +106,12 @@ func testRunInterruptedJob(t *testing.T, d *ddl, job *model.Job) { } func TestSchemaResume(t *testing.T) { - store := createMockStore(t) - defer func() { - require.NoError(t, store.Close()) - }() + store, dom, clean := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease) + defer clean() - d1, err := testNewDDLAndStart( - context.Background(), - WithStore(store), - WithLease(testLease), - ) - require.NoError(t, err) - defer func() { - require.NoError(t, d1.Stop()) - }() + require.True(t, dom.DDL().OwnerManager().IsOwner()) - require.True(t, d1.OwnerManager().IsOwner()) - - dbInfo, err := testSchemaInfo(d1, "test_restart") + dbInfo, err := testSchemaInfo(store, "test_restart") require.NoError(t, err) job := &model.Job{ SchemaID: dbInfo.ID, @@ -140,37 +119,25 @@ func TestSchemaResume(t *testing.T) { BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{dbInfo}, } - testRunInterruptedJob(t, d1, job) - testCheckSchemaState(t, d1, dbInfo, model.StatePublic) + testRunInterruptedJob(t, store, dom, job) + testCheckSchemaState(t, store, dbInfo, model.StatePublic) job = &model.Job{ SchemaID: dbInfo.ID, Type: model.ActionDropSchema, BinlogInfo: &model.HistoryInfo{}, } - testRunInterruptedJob(t, d1, job) - testCheckSchemaState(t, d1, dbInfo, model.StateNone) + testRunInterruptedJob(t, store, dom, job) + testCheckSchemaState(t, store, dbInfo, model.StateNone) } func TestStat(t *testing.T) { - store := createMockStore(t) - defer func() { - require.NoError(t, store.Close()) - }() - - d, err := testNewDDLAndStart( - context.Background(), - WithStore(store), - WithLease(testLease), - ) - require.NoError(t, err) - defer func() { - require.NoError(t, d.Stop()) - }() + store, dom, clean := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease) + defer clean() - dbInfo, err := testSchemaInfo(d, "test_restart") + dbInfo, err := testSchemaInfo(store, "test_restart") require.NoError(t, err) - testCreateSchema(t, testNewContext(d), d, dbInfo) + testCreateSchema(t, testkit.NewTestKit(t, store).Session(), dom.DDL(), dbInfo) // TODO: Get this information from etcd. // m, err := d.Stats(nil) @@ -185,19 +152,17 @@ func TestStat(t *testing.T) { } done := make(chan error, 1) - go runInterruptedJob(d, job, done) + go runInterruptedJob(t, store, dom.DDL(), job, done) - ticker := time.NewTicker(d.lease * 1) + ticker := time.NewTicker(dom.DDL().GetLease() * 1) defer ticker.Stop() - ver := getDDLSchemaVer(t, d) + ver := getDDLSchemaVer(t, dom.DDL()) LOOP: for { select { case <-ticker.C: - err := d.Stop() - require.Nil(t, err) - require.GreaterOrEqual(t, getDDLSchemaVer(t, d), ver) - d.restartWorkers(context.Background()) + require.GreaterOrEqual(t, getDDLSchemaVer(t, dom.DDL()), ver) + restartWorkers(t, store, dom) time.Sleep(time.Millisecond * 20) case err := <-done: // TODO: Get this information from etcd. @@ -209,31 +174,19 @@ LOOP: } func TestTableResume(t *testing.T) { - store := createMockStore(t) - defer func() { - require.NoError(t, store.Close()) - }() - - d, err := testNewDDLAndStart( - context.Background(), - WithStore(store), - WithLease(testLease), - ) - require.NoError(t, err) - defer func() { - require.NoError(t, d.Stop()) - }() + store, dom, clean := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease) + defer clean() - dbInfo, err := testSchemaInfo(d, "test_table") + dbInfo, err := testSchemaInfo(store, "test_table") require.NoError(t, err) - testCreateSchema(t, testNewContext(d), d, dbInfo) + testCreateSchema(t, testkit.NewTestKit(t, store).Session(), dom.DDL(), dbInfo) defer func() { - testDropSchema(t, testNewContext(d), d, dbInfo) + testDropSchema(t, testkit.NewTestKit(t, store).Session(), dom.DDL(), dbInfo) }() - require.True(t, d.OwnerManager().IsOwner()) + require.True(t, dom.DDL().OwnerManager().IsOwner()) - tblInfo, err := testTableInfo(d, "t1", 3) + tblInfo, err := testTableInfo(store, "t1", 3) require.NoError(t, err) job := &model.Job{ SchemaID: dbInfo.ID, @@ -242,8 +195,8 @@ func TestTableResume(t *testing.T) { BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{tblInfo}, } - testRunInterruptedJob(t, d, job) - testCheckTableState(t, d, dbInfo, tblInfo, model.StatePublic) + testRunInterruptedJob(t, store, dom, job) + testCheckTableState(t, store, dbInfo, tblInfo, model.StatePublic) job = &model.Job{ SchemaID: dbInfo.ID, @@ -251,55 +204,6 @@ func TestTableResume(t *testing.T) { Type: model.ActionDropTable, BinlogInfo: &model.HistoryInfo{}, } - testRunInterruptedJob(t, d, job) - testCheckTableState(t, d, dbInfo, tblInfo, model.StateNone) -} - -// testTableInfo creates a test table with num int columns and with no index. -func testTableInfo(d *ddl, name string, num int) (*model.TableInfo, error) { - tblInfo := &model.TableInfo{ - Name: model.NewCIStr(name), - } - genIDs, err := d.genGlobalIDs(1) - - if err != nil { - return nil, err - } - tblInfo.ID = genIDs[0] - - cols := make([]*model.ColumnInfo, num) - for i := range cols { - col := &model.ColumnInfo{ - Name: model.NewCIStr(fmt.Sprintf("c%d", i+1)), - Offset: i, - DefaultValue: i + 1, - State: model.StatePublic, - } - - col.FieldType = *types.NewFieldType(mysql.TypeLong) - col.ID = allocateColumnID(tblInfo) - cols[i] = col - } - tblInfo.Columns = cols - tblInfo.Charset = "utf8" - tblInfo.Collate = "utf8_bin" - return tblInfo, nil -} - -func testCheckTableState(t *testing.T, d *ddl, dbInfo *model.DBInfo, tblInfo *model.TableInfo, state model.SchemaState) { - ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) - require.NoError(t, kv.RunInNewTxn(ctx, d.store, false, func(ctx context.Context, txn kv.Transaction) error { - m := meta.NewMeta(txn) - info, err := m.GetTable(dbInfo.ID, tblInfo.ID) - require.NoError(t, err) - - if state == model.StateNone { - require.NoError(t, err) - return nil - } - - require.Equal(t, info.Name, tblInfo.Name) - require.Equal(t, info.State, state) - return nil - })) + testRunInterruptedJob(t, store, dom, job) + testCheckTableState(t, store, dbInfo, tblInfo, model.StateNone) } From 89976045314f52c77c96c690cc7432ab14edb50e Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Wed, 13 Jul 2022 18:59:05 +0800 Subject: [PATCH 16/27] store/mockstore/unistore: fix several issues of coprocessor paging in unistore (#36147) close pingcap/tidb#36145 --- .../unistore/cophandler/cop_handler.go | 3 ++- store/mockstore/unistore/cophandler/mpp.go | 25 ++++++++++++------- .../mockstore/unistore/cophandler/mpp_exec.go | 15 ++++++++++- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/store/mockstore/unistore/cophandler/cop_handler.go b/store/mockstore/unistore/cophandler/cop_handler.go index 75fa686ff8fca..5f375f2bfdc30 100644 --- a/store/mockstore/unistore/cophandler/cop_handler.go +++ b/store/mockstore/unistore/cophandler/cop_handler.go @@ -179,6 +179,7 @@ func buildAndRunMPPExecutor(dagCtx *dagContext, dagReq *tipb.DAGRequest, pagingS if pagingSize > 0 { lastRange = &coprocessor.KeyRange{} builder.paging = lastRange + builder.pagingSize = pagingSize } exec, err := builder.buildMPPExecutor(rootExec) if err != nil { @@ -221,7 +222,7 @@ func mppExecute(exec mppExec, dagCtx *dagContext, dagReq *tipb.DAGRequest, pagin if pagingSize > 0 { totalRows += uint64(chk.NumRows()) if totalRows > pagingSize { - break + return } } default: diff --git a/store/mockstore/unistore/cophandler/mpp.go b/store/mockstore/unistore/cophandler/mpp.go index fb9cfeaf1aff1..1cf0746e861dd 100644 --- a/store/mockstore/unistore/cophandler/mpp.go +++ b/store/mockstore/unistore/cophandler/mpp.go @@ -51,14 +51,15 @@ const ( ) type mppExecBuilder struct { - sc *stmtctx.StatementContext - dbReader *dbreader.DBReader - mppCtx *MPPCtx - dagReq *tipb.DAGRequest - dagCtx *dagContext - counts []int64 - ndvs []int64 - paging *coprocessor.KeyRange + sc *stmtctx.StatementContext + dbReader *dbreader.DBReader + mppCtx *MPPCtx + dagReq *tipb.DAGRequest + dagCtx *dagContext + counts []int64 + ndvs []int64 + paging *coprocessor.KeyRange + pagingSize uint64 } func (b *mppExecBuilder) buildMPPTableScan(pb *tipb.TableScan) (*tableScanExec, error) { @@ -199,7 +200,7 @@ func (b *mppExecBuilder) buildLimit(pb *tipb.Limit) (*limitExec, error) { return exec, nil } -func (b *mppExecBuilder) buildTopN(pb *tipb.TopN) (*topNExec, error) { +func (b *mppExecBuilder) buildTopN(pb *tipb.TopN) (mppExec, error) { child, err := b.buildMPPExecutor(pb.Child) if err != nil { return nil, err @@ -227,6 +228,12 @@ func (b *mppExecBuilder) buildTopN(pb *tipb.TopN) (*topNExec, error) { row: newTopNSortRow(len(conds)), topn: pb.Limit, } + + // When using paging protocol, if paging size < topN limit, the topN exec degenerate to do nothing. + if b.paging != nil && b.pagingSize < pb.Limit { + exec.dummy = true + } + return exec, nil } diff --git a/store/mockstore/unistore/cophandler/mpp_exec.go b/store/mockstore/unistore/cophandler/mpp_exec.go index 8e079991c7daa..3e164da24d458 100644 --- a/store/mockstore/unistore/cophandler/mpp_exec.go +++ b/store/mockstore/unistore/cophandler/mpp_exec.go @@ -319,7 +319,8 @@ func (e *indexScanExec) Process(key, value []byte) error { if e.chk.IsFull() { e.chunks = append(e.chunks, e.chk) if e.paging != nil { - e.chunkLastProcessedKeys = append(e.chunkLastProcessedKeys, key) + lastProcessed := kv.Key(append([]byte{}, key...)) // need a deep copy to store the key + e.chunkLastProcessedKeys = append(e.chunkLastProcessedKeys, lastProcessed) } e.chk = chunk.NewChunkWithCapacity(e.fieldTypes, DefaultBatchSize) } @@ -423,6 +424,9 @@ type topNExec struct { conds []expression.Expression row *sortRow recv []*chunk.Chunk + + // When dummy is true, topNExec just copy what it read from children to its parent. + dummy bool } func (e *topNExec) open() error { @@ -432,6 +436,11 @@ func (e *topNExec) open() error { if err != nil { return err } + + if e.dummy { + return nil + } + for { chk, err = e.children[0].next() if err != nil { @@ -466,6 +475,10 @@ func (e *topNExec) open() error { } func (e *topNExec) next() (*chunk.Chunk, error) { + if e.dummy { + return e.children[0].next() + } + chk := chunk.NewChunkWithCapacity(e.getFieldTypes(), DefaultBatchSize) for ; !chk.IsFull() && e.idx < e.topn && e.idx < uint64(e.heap.heapSize); e.idx++ { row := e.heap.rows[e.idx] From 2ff12e8b646de1ccbf64618259feb8f31d2a29cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=B1=E5=B2=9A?= <36239017+YuJuncen@users.noreply.github.com> Date: Wed, 13 Jul 2022 19:25:05 +0800 Subject: [PATCH 17/27] log-backup: fix checkpoint display (#36166) fixed pingcap/tidb#36092 --- br/pkg/stream/stream_status.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/br/pkg/stream/stream_status.go b/br/pkg/stream/stream_status.go index e08f3f6c34513..53ab1a0fc2499 100644 --- a/br/pkg/stream/stream_status.go +++ b/br/pkg/stream/stream_status.go @@ -136,6 +136,8 @@ func (p *printByTable) AddTask(task TaskStatus) { info := fmt.Sprintf("%s; gap=%s", pTime, gapColor.Sprint(gap)) return info } + cp := task.GetMinStoreCheckpoint() + table.Add("checkpoint[global]", formatTS(cp.TS)) p.addCheckpoints(&task, table, formatTS) for store, e := range task.LastErrors { table.Add(fmt.Sprintf("error[store=%d]", store), e.ErrorCode) @@ -147,21 +149,15 @@ func (p *printByTable) AddTask(task TaskStatus) { func (p *printByTable) addCheckpoints(task *TaskStatus, table *glue.Table, formatTS func(uint64) string) { cp := task.GetMinStoreCheckpoint() - items := make([][2]string, 0, len(task.Checkpoints)) if cp.Type() != CheckpointTypeGlobal { for _, cp := range task.Checkpoints { switch cp.Type() { case CheckpointTypeStore: - items = append(items, [2]string{fmt.Sprintf("checkpoint[store=%d]", cp.ID), formatTS(cp.TS)}) + table.Add(fmt.Sprintf("checkpoint[store=%d]", cp.ID), formatTS(cp.TS)) } } - } else { - items = append(items, [2]string{"checkpoint[central-global]", formatTS(cp.TS)}) } - for _, item := range items { - table.Add(item[0], item[1]) - } } func (p *printByTable) PrintTasks() { From 28c96008a9af3bb11f66057b656a4f20396739d9 Mon Sep 17 00:00:00 2001 From: zyguan Date: Wed, 13 Jul 2022 19:45:05 +0800 Subject: [PATCH 18/27] executor: check the error returned by `handleNoDelay` (#36105) close pingcap/tidb#35105 --- executor/adapter.go | 2 +- executor/explain_test.go | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/executor/adapter.go b/executor/adapter.go index b1a759cc1be27..1ac4ecef0194d 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -469,7 +469,7 @@ func (a *ExecStmt) Exec(ctx context.Context) (_ sqlexec.RecordSet, err error) { return a.handlePessimisticSelectForUpdate(ctx, e) } - if handled, result, err := a.handleNoDelay(ctx, e, isPessimistic); handled { + if handled, result, err := a.handleNoDelay(ctx, e, isPessimistic); handled || err != nil { return result, err } diff --git a/executor/explain_test.go b/executor/explain_test.go index af3439b048b81..8ecbd3edd019b 100644 --- a/executor/explain_test.go +++ b/executor/explain_test.go @@ -495,3 +495,16 @@ func TestIssue35911(t *testing.T) { // To be consistent with other operators, we should not aggregate the concurrency in the runtime stats. require.EqualValues(t, 5, concurrency) } + +func TestIssue35105(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int primary key)") + tk.MustExec("insert into t values (2)") + tk.MustExec("set @@tidb_constraint_check_in_place=1") + require.Error(t, tk.ExecToErr("explain analyze insert into t values (1), (2), (3)")) + tk.MustQuery("select * from t").Check(testkit.Rows("2")) +} From f2823250fb937d6ea3778602cf97a0e356391328 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 13 Jul 2022 20:07:05 +0800 Subject: [PATCH 19/27] domain: fix unstable test TestAbnormalSessionPool (#36154) close pingcap/tidb#36153 --- .bazelrc | 1 - distsql/BUILD.bazel | 1 + domain/db_test.go | 6 ++---- domain/infosync/BUILD.bazel | 1 + domain/main_test.go | 2 ++ 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.bazelrc b/.bazelrc index 2a1b030700ec2..d622cdfe057ed 100644 --- a/.bazelrc +++ b/.bazelrc @@ -5,7 +5,6 @@ build --java_language_version=17 build --java_runtime_version=17 build --tool_java_language_version=17 build --tool_java_runtime_version=17 -build --experimental_remote_cache_compression run --color=yes build:release --workspace_status_command=./build/print-workspace-status.sh --stamp diff --git a/distsql/BUILD.bazel b/distsql/BUILD.bazel index 9f20df059fe06..822a49dd79e39 100644 --- a/distsql/BUILD.bazel +++ b/distsql/BUILD.bazel @@ -53,6 +53,7 @@ go_library( go_test( name = "distsql_test", + timeout = "short", srcs = [ "bench_test.go", "distsql_test.go", diff --git a/domain/db_test.go b/domain/db_test.go index 25b24d8116604..cdd02c949404c 100644 --- a/domain/db_test.go +++ b/domain/db_test.go @@ -76,8 +76,7 @@ func TestNormalSessionPool(t *testing.T) { info, err1 := infosync.GlobalInfoSyncerInit(context.Background(), "t", func() uint64 { return 1 }, nil, true) require.NoError(t, err1) conf := config.GetGlobalConfig() - conf.Port = 0 - conf.Status.StatusPort = 10045 + conf.Socket = "" svr, err := server.NewServer(conf, nil) require.NoError(t, err) svr.SetDomain(domain) @@ -109,8 +108,7 @@ func TestAbnormalSessionPool(t *testing.T) { info, err1 := infosync.GlobalInfoSyncerInit(context.Background(), "t", func() uint64 { return 1 }, nil, true) require.NoError(t, err1) conf := config.GetGlobalConfig() - conf.Port = 0 - conf.Status.StatusPort = 10046 + conf.Socket = "" svr, err := server.NewServer(conf, nil) require.NoError(t, err) svr.SetDomain(domain) diff --git a/domain/infosync/BUILD.bazel b/domain/infosync/BUILD.bazel index e9ecff7fa5606..c92426d835c2f 100644 --- a/domain/infosync/BUILD.bazel +++ b/domain/infosync/BUILD.bazel @@ -47,6 +47,7 @@ go_library( go_test( name = "infosync_test", + timeout = "short", srcs = ["info_test.go"], embed = [":infosync"], flaky = True, diff --git a/domain/main_test.go b/domain/main_test.go index fc4bc11227206..163fedbad111a 100644 --- a/domain/main_test.go +++ b/domain/main_test.go @@ -17,11 +17,13 @@ package domain_test import ( "testing" + "github.com/pingcap/tidb/server" "github.com/pingcap/tidb/testkit/testsetup" "go.uber.org/goleak" ) func TestMain(m *testing.M) { + server.RunInGoTest = true testsetup.SetupForCommonTest() opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), From d93bc7a4b03ba1513529c738f10f7998e9c6e18c Mon Sep 17 00:00:00 2001 From: niubell Date: Wed, 13 Jul 2022 20:31:05 +0800 Subject: [PATCH 20/27] table-filter: optimize table pattern message and unit tests (#36160) close pingcap/tidb#36163 --- util/table-filter/parser.go | 2 +- util/table-filter/table_filter_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/util/table-filter/parser.go b/util/table-filter/parser.go index 122984f95f86d..d30ddb7a17b48 100644 --- a/util/table-filter/parser.go +++ b/util/table-filter/parser.go @@ -58,7 +58,7 @@ func (p *tableRulesParser) parse(line string, canImport bool) error { return err } if len(line) == 0 { - return p.errorf("missing table pattern") + return p.errorf("wrong table pattern") } if line[0] != '.' { return p.errorf("syntax error: missing '.' between schema and table patterns") diff --git a/util/table-filter/table_filter_test.go b/util/table-filter/table_filter_test.go index 7e0eb8956239e..2928d498098ea 100644 --- a/util/table-filter/table_filter_test.go +++ b/util/table-filter/table_filter_test.go @@ -265,8 +265,8 @@ func TestMatchTables(t *testing.T) { require.NoError(t, err) fci := filter.CaseInsensitive(fcs) for i, tbl := range tc.tables { - require.Equalf(t, fcs.MatchTable(tbl.Schema, tbl.Name), tc.acceptedCS[i], "cs tbl %v", tbl) - require.Equalf(t, fci.MatchTable(tbl.Schema, tbl.Name), tc.acceptedCI[i], "ci tbl %v", tbl) + require.Equalf(t, tc.acceptedCS[i], fcs.MatchTable(tbl.Schema, tbl.Name), "cs tbl %v", tbl) + require.Equalf(t, tc.acceptedCI[i], fci.MatchTable(tbl.Schema, tbl.Name), "ci tbl %v", tbl) } } } @@ -383,7 +383,7 @@ func TestParseFailures2(t *testing.T) { }, { arg: "db", - msg: `.*: missing table pattern`, + msg: `.*: wrong table pattern`, }, { arg: "db.", From 2f934d67a2381a482403dcfad65afeea9ac3274b Mon Sep 17 00:00:00 2001 From: tangenta Date: Wed, 13 Jul 2022 20:57:05 +0800 Subject: [PATCH 21/27] *: support show ddl jobs for sub-jobs (#36168) ref pingcap/tidb#14766 --- ddl/multi_schema_change_test.go | 35 ++++++++++++++++++++++++++++++ executor/executor.go | 16 ++++++++++++++ executor/infoschema_reader.go | 8 +++++++ executor/infoschema_reader_test.go | 5 +++++ parser/model/ddl.go | 6 ++++- 5 files changed, 69 insertions(+), 1 deletion(-) diff --git a/ddl/multi_schema_change_test.go b/ddl/multi_schema_change_test.go index eeede7a8ea954..8d688ea9ff483 100644 --- a/ddl/multi_schema_change_test.go +++ b/ddl/multi_schema_change_test.go @@ -891,6 +891,41 @@ func TestMultiSchemaChangeModifyColumnsCancelled(t *testing.T) { Check(testkit.Rows("int")) } +func TestMultiSchemaChangeAdminShowDDLJobs(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + originHook := dom.DDL().GetHook() + hook := &ddl.TestDDLCallback{Do: dom} + hook.OnJobRunBeforeExported = func(job *model.Job) { + assert.Equal(t, model.ActionMultiSchemaChange, job.Type) + if job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateDeleteOnly { + newTk := testkit.NewTestKit(t, store) + rows := newTk.MustQuery("admin show ddl jobs 1").Rows() + // 1 history job and 1 running job with 2 subjobs + assert.Equal(t, len(rows), 4) + assert.Equal(t, rows[1][1], "test") + assert.Equal(t, rows[1][2], "t") + assert.Equal(t, rows[1][3], "add index /* subjob */") + assert.Equal(t, rows[1][4], "delete only") + assert.Equal(t, rows[1][len(rows[1])-1], "running") + + assert.Equal(t, rows[2][3], "add index /* subjob */") + assert.Equal(t, rows[2][4], "none") + assert.Equal(t, rows[2][len(rows[2])-1], "queueing") + } + } + + tk.MustExec("create table t (a int, b int, c int)") + tk.MustExec("insert into t values (1, 2, 3)") + + dom.DDL().SetHook(hook) + tk.MustExec("alter table t add index t(a), add index t1(b)") + dom.DDL().SetHook(originHook) +} + func TestMultiSchemaChangeWithExpressionIndex(t *testing.T) { store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() diff --git a/executor/executor.go b/executor/executor.go index 7d2839cb29e34..be6e75495fce0 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -568,6 +568,22 @@ func (e *DDLJobRetriever) appendJobToChunk(req *chunk.Chunk, job *model.Job, che req.AppendNull(10) } req.AppendString(11, job.State.String()) + if job.Type == model.ActionMultiSchemaChange { + for _, subJob := range job.MultiSchemaInfo.SubJobs { + req.AppendInt64(0, job.ID) + req.AppendString(1, schemaName) + req.AppendString(2, tableName) + req.AppendString(3, subJob.Type.String()+" /* subjob */") + req.AppendString(4, subJob.SchemaState.String()) + req.AppendInt64(5, job.SchemaID) + req.AppendInt64(6, job.TableID) + req.AppendInt64(7, subJob.RowCount) + req.AppendNull(8) + req.AppendNull(9) + req.AppendNull(10) + req.AppendString(11, subJob.State.String()) + } + } } func ts2Time(timestamp uint64, loc *time.Location) types.Time { diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index 6adcf3e01f851..bd25f2825cd69 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -1303,6 +1303,9 @@ func (e *DDLJobsReaderExec) Next(ctx context.Context, req *chunk.Chunk) error { for i := e.cursor; i < e.cursor+num; i++ { e.appendJobToChunk(req, e.runningJobs[i], checker) req.AppendString(12, e.runningJobs[i].Query) + for range e.runningJobs[i].MultiSchemaInfo.SubJobs { + req.AppendString(12, e.runningJobs[i].Query) + } } e.cursor += num count += num @@ -1318,6 +1321,11 @@ func (e *DDLJobsReaderExec) Next(ctx context.Context, req *chunk.Chunk) error { for _, job := range e.cacheJobs { e.appendJobToChunk(req, job, checker) req.AppendString(12, job.Query) + if job.Type == model.ActionMultiSchemaChange { + for range job.MultiSchemaInfo.SubJobs { + req.AppendString(12, job.Query) + } + } } e.cursor += len(e.cacheJobs) } diff --git a/executor/infoschema_reader_test.go b/executor/infoschema_reader_test.go index 3f5631d43bff1..97531b2489140 100644 --- a/executor/infoschema_reader_test.go +++ b/executor/infoschema_reader_test.go @@ -243,6 +243,11 @@ func TestDDLJobs(t *testing.T) { DDLJobsTester.MustExec("set role r_priv") DDLJobsTester.MustQuery("select DB_NAME, TABLE_NAME from information_schema.DDL_JOBS where DB_NAME = 'test_ddl_jobs' and TABLE_NAME = 't';").Check( testkit.Rows("test_ddl_jobs t")) + + tk.MustExec("create table tt (a int);") + tk.MustExec("alter table tt add index t(a), add column b int") + tk.MustQuery("select db_name, table_name, job_type from information_schema.DDL_JOBS limit 3").Check( + testkit.Rows("test_ddl_jobs tt alter table multi-schema change", "test_ddl_jobs tt add index /* subjob */", "test_ddl_jobs tt add column /* subjob */")) } func TestKeyColumnUsage(t *testing.T) { diff --git a/parser/model/ddl.go b/parser/model/ddl.go index 6be7ad22c571f..62f2fe5239e62 100644 --- a/parser/model/ddl.go +++ b/parser/model/ddl.go @@ -554,8 +554,12 @@ func (job *Job) DecodeArgs(args ...interface{}) error { // String implements fmt.Stringer interface. func (job *Job) String() string { rowCount := job.GetRowCount() - return fmt.Sprintf("ID:%d, Type:%s, State:%s, SchemaState:%s, SchemaID:%d, TableID:%d, RowCount:%d, ArgLen:%d, start time: %v, Err:%v, ErrCount:%d, SnapshotVersion:%v", + ret := fmt.Sprintf("ID:%d, Type:%s, State:%s, SchemaState:%s, SchemaID:%d, TableID:%d, RowCount:%d, ArgLen:%d, start time: %v, Err:%v, ErrCount:%d, SnapshotVersion:%v", job.ID, job.Type, job.State, job.SchemaState, job.SchemaID, job.TableID, rowCount, len(job.Args), TSConvert2Time(job.StartTS), job.Error, job.ErrorCount, job.SnapshotVer) + if job.Type != ActionMultiSchemaChange && job.MultiSchemaInfo != nil { + ret += fmt.Sprintf(", Multi-Schema Change:true, Revertible:%v", job.MultiSchemaInfo.Revertible) + } + return ret } func (job *Job) hasDependentSchema(other *Job) (bool, error) { From 24eb419ca2da36cf689fe9923380a2cea8a32227 Mon Sep 17 00:00:00 2001 From: Zhou Kunqin <25057648+time-and-fate@users.noreply.github.com> Date: Wed, 13 Jul 2022 21:59:05 +0800 Subject: [PATCH 22/27] planner: handle the expected row count for pushed-down selection in mpp (#36195) close pingcap/tidb#36194 --- planner/core/find_best_task.go | 2 +- planner/core/integration_test.go | 28 +++++++++++++++++++ .../core/testdata/integration_suite_out.json | 6 ++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 13c559d9c66c5..00c5099388e08 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1917,7 +1917,7 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid ColumnNames: ds.names, } ts.cost = cost - mppTask = ts.addPushedDownSelectionToMppTask(mppTask, ds.stats) + mppTask = ts.addPushedDownSelectionToMppTask(mppTask, ds.stats.ScaleByExpectCnt(prop.ExpectedCnt)) return mppTask, nil } copTask := &copTask{ diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index e059d97711560..49a8b012705f6 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -6989,3 +6989,31 @@ func TestIssue25813(t *testing.T) { " └─TableReader(Probe) 10000.00 root data:TableFullScan", " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo")) } + +func TestIssue36194(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int)") + // create virtual tiflash replica. + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + require.True(t, exists) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "t" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + tk.MustQuery("explain format = 'brief' select * from t where a + 1 > 20 limit 100;;").Check(testkit.Rows( + "Limit 100.00 root offset:0, count:100", + "└─TableReader 100.00 root data:ExchangeSender", + " └─ExchangeSender 100.00 mpp[tiflash] ExchangeType: PassThrough", + " └─Limit 100.00 mpp[tiflash] offset:0, count:100", + " └─Selection 100.00 mpp[tiflash] gt(plus(test.t.a, 1), 20)", + " └─TableFullScan 125.00 mpp[tiflash] table:t keep order:false, stats:pseudo")) +} diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index aeff80fd103ea..22e3d3d4f582d 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -5952,7 +5952,7 @@ " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.id))", " │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", - " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.id))", + " └─Selection(Probe) 0.80 mpp[tiflash] not(isnull(test.t.id))", " └─TableFullScan 0.80 mpp[tiflash] table:t1 keep order:false, stats:pseudo" ] }, @@ -5968,7 +5968,7 @@ " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.id))", " │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", - " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.id))", + " └─Selection(Probe) 0.80 mpp[tiflash] not(isnull(test.t.id))", " └─TableFullScan 0.80 mpp[tiflash] table:t1 keep order:false, stats:pseudo" ] }, @@ -5985,7 +5985,7 @@ " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.id))", " │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", - " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.id))", + " └─Selection(Probe) 16.00 mpp[tiflash] not(isnull(test.t.id))", " └─TableFullScan 16.02 mpp[tiflash] table:t1 keep order:false, stats:pseudo" ] } From 0b427e1fd60503dc58ab15b82cd9dff975655813 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Wed, 13 Jul 2022 22:27:05 +0800 Subject: [PATCH 23/27] *: add tidb_min_paging_size system variable (#36107) close pingcap/tidb#36106 --- distsql/request_builder.go | 1 + executor/distsql_test.go | 59 ++++++++++++++++++++++++++++ kv/kv.go | 2 + sessionctx/variable/session.go | 4 ++ sessionctx/variable/sysvar.go | 5 +++ sessionctx/variable/tidb_vars.go | 5 +++ sessionctx/variable/varsutil_test.go | 8 ++++ store/copr/coprocessor.go | 3 +- store/copr/coprocessor_test.go | 1 + 9 files changed, 87 insertions(+), 1 deletion(-) diff --git a/distsql/request_builder.go b/distsql/request_builder.go index 2bd7c5df65c04..e5d1e23f437a0 100644 --- a/distsql/request_builder.go +++ b/distsql/request_builder.go @@ -266,6 +266,7 @@ func (builder *RequestBuilder) SetFromSessionVars(sv *variable.SessionVars) *Req builder.SetResourceGroupTagger(sv.StmtCtx.GetResourceGroupTagger()) if sv.EnablePaging { builder.SetPaging(true) + builder.Request.MinPagingSize = uint64(sv.MinPagingSize) } builder.RequestSource.RequestSourceInternal = sv.InRestrictedSQL builder.RequestSource.RequestSourceType = sv.RequestSourceType diff --git a/executor/distsql_test.go b/executor/distsql_test.go index 71156d890a9a9..32cb9818e2452 100644 --- a/executor/distsql_test.go +++ b/executor/distsql_test.go @@ -19,7 +19,9 @@ import ( "context" "fmt" "math/rand" + "regexp" "runtime/pprof" + "strconv" "strings" "testing" "time" @@ -34,6 +36,7 @@ import ( "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/mock" + "github.com/pingcap/tidb/util/paging" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/testutils" ) @@ -418,3 +421,59 @@ func TestPartitionTableIndexJoinIndexLookUp(t *testing.T) { tk.MustQuery("select /*+ TIDB_INLJ(t1, t2) */ t1.* from t t1, t t2 use index(a) where t1.a=t2.b and " + cond).Sort().Check(result) } } + +func TestCoprocessorPagingSize(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + + tk.MustExec("use test") + tk.MustExec("create table t_paging (a int, b int, key(a), key(b))") + nRows := 512 + values := make([]string, 0, nRows) + for i := 0; i < nRows; i++ { + values = append(values, fmt.Sprintf("(%v, %v)", rand.Intn(nRows), rand.Intn(nRows))) + } + tk.MustExec(fmt.Sprintf("insert into t_paging values %v", strings.Join(values, ", "))) + tk.MustQuery("select @@tidb_min_paging_size").Check(testkit.Rows(strconv.FormatUint(paging.MinPagingSize, 10))) + + // When the min paging size is small, we need more RPC roundtrip! + // Check 'rpc_num' in the execution information + // + // mysql> explain analyze select * from t_paging; + // +--------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + // | id |task | execution info | + // +--------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + // | TableReader_5 |root | time:7.27ms, loops:2, cop_task: {num: 10, max: 1.57ms, min: 313.3µs, avg: 675.9µs, p95: 1.57ms, tot_proc: 2ms, rpc_num: 10, rpc_time: 6.69ms, copr_cache_hit_ratio: 0.00, distsql_concurrency: 15} | + // | └─TableFullScan_4 |cop[tikv] | tikv_task:{proc max:1.48ms, min:294µs, avg: 629µs, p80:1.21ms, p95:1.48ms, iters:0, tasks:10} | + // +--------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + // 2 rows in set (0.01 sec) + + getRPCNumFromExplain := func(rows [][]interface{}) (res uint64) { + re := regexp.MustCompile("rpc_num: ([0-9]+)") + for _, row := range rows { + buf := bytes.NewBufferString("") + _, _ = fmt.Fprintf(buf, "%s\n", row) + if matched := re.FindStringSubmatch(buf.String()); matched != nil { + require.Equal(t, len(matched), 2) + c, err := strconv.ParseUint(matched[1], 10, 64) + require.NoError(t, err) + return c + } + } + return res + } + + // This is required here because only the chunk encoding collect the execution information and contains 'rpc_num'. + tk.MustExec("set @@tidb_enable_chunk_rpc = on") + + tk.MustExec("set @@tidb_min_paging_size = 1") + rows := tk.MustQuery("explain analyze select * from t_paging").Rows() + rpcNum := getRPCNumFromExplain(rows) + require.Greater(t, rpcNum, uint64(2)) + + tk.MustExec("set @@tidb_min_paging_size = 1000") + rows = tk.MustQuery("explain analyze select * from t_paging").Rows() + rpcNum = getRPCNumFromExplain(rows) + require.Equal(t, rpcNum, uint64(1)) +} diff --git a/kv/kv.go b/kv/kv.go index d65d6498c12d1..5481d57ea0696 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -368,6 +368,8 @@ type Request struct { ResourceGroupTagger tikvrpc.ResourceGroupTagger // Paging indicates whether the request is a paging request. Paging bool + // MinPagingSize is used when Paging is true. + MinPagingSize uint64 // RequestSource indicates whether the request is an internal request. RequestSource util.RequestSource } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index add261a90d449..54eb0a55de423 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1433,6 +1433,7 @@ func NewSessionVars() *SessionVars { IndexLookupSize: DefIndexLookupSize, InitChunkSize: DefInitChunkSize, MaxChunkSize: DefMaxChunkSize, + MinPagingSize: DefMinPagingSize, } vars.DMLBatchSize = DefDMLBatchSize vars.AllowBatchCop = DefTiDBAllowBatchCop @@ -2173,6 +2174,9 @@ type BatchSize struct { // MaxChunkSize defines max row count of a Chunk during query execution. MaxChunkSize int + + // MinPagingSize defines the min size used by the coprocessor paging protocol. + MinPagingSize int } const ( diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index dba7781dfbe1f..fe83218698719 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -35,6 +35,7 @@ import ( "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/mathutil" + "github.com/pingcap/tidb/util/paging" "github.com/pingcap/tidb/util/stmtsummary" "github.com/pingcap/tidb/util/tikvutil" "github.com/pingcap/tidb/util/tls" @@ -1667,6 +1668,10 @@ var defaultSysVars = []*SysVar{ metrics.ToggleSimplifiedMode(TiDBOptOn(s)) return nil }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBMinPagingSize, Value: strconv.Itoa(DefMinPagingSize), Type: TypeUnsigned, MinValue: 1, MaxValue: paging.MaxPagingSize, SetSession: func(s *SessionVars, val string) error { + s.MinPagingSize = tidbOptPositiveInt32(val, DefMinPagingSize) + return nil + }}, {Scope: ScopeSession, Name: TiDBMemoryDebugModeMinHeapInUse, Value: strconv.Itoa(0), Type: TypeInt, MinValue: math.MinInt64, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { s.MemoryDebugModeMinHeapInUse = TidbOptInt64(val, 0) return nil diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 54b49f840f494..ff3cfdb41dd4a 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/parser/mysql" + "github.com/pingcap/tidb/util/paging" "go.uber.org/atomic" ) @@ -363,6 +364,9 @@ const ( // TiDBInitChunkSize is used to control the init chunk size during query execution. TiDBInitChunkSize = "tidb_init_chunk_size" + // TiDBMinPagingSize is used to control the min paging size in the coprocessor paging protocol. + TiDBMinPagingSize = "tidb_min_paging_size" + // TiDBEnableCascadesPlanner is used to control whether to enable the cascades planner. TiDBEnableCascadesPlanner = "tidb_enable_cascades_planner" @@ -818,6 +822,7 @@ const ( DefBatchCommit = false DefCurretTS = 0 DefInitChunkSize = 32 + DefMinPagingSize = int(paging.MinPagingSize) DefMaxChunkSize = 1024 DefDMLBatchSize = 0 DefMaxPreparedStmtCount = -1 diff --git a/sessionctx/variable/varsutil_test.go b/sessionctx/variable/varsutil_test.go index 4641a8c2f1e0d..49362e0fc3969 100644 --- a/sessionctx/variable/varsutil_test.go +++ b/sessionctx/variable/varsutil_test.go @@ -432,6 +432,14 @@ func TestVarsutil(t *testing.T) { err = SetSessionSystemVar(v, TiDBTableCacheLease, "123") require.Error(t, err) require.Regexp(t, "'tidb_table_cache_lease' is a GLOBAL variable and should be set with SET GLOBAL", err.Error()) + + val, err = GetSessionOrGlobalSystemVar(v, TiDBMinPagingSize) + require.NoError(t, err) + require.Equal(t, strconv.Itoa(DefMinPagingSize), val) + + err = SetSessionSystemVar(v, TiDBMinPagingSize, "123") + require.NoError(t, err) + require.Equal(t, v.MinPagingSize, 123) } func TestValidate(t *testing.T) { diff --git a/store/copr/coprocessor.go b/store/copr/coprocessor.go index 29cc437b182e6..e7529fe6a5201 100644 --- a/store/copr/coprocessor.go +++ b/store/copr/coprocessor.go @@ -211,7 +211,7 @@ func buildCopTasks(bo *Backoffer, cache *RegionCache, ranges *KeyRanges, req *kv // the size will grow every round. pagingSize := uint64(0) if req.Paging { - pagingSize = paging.MinPagingSize + pagingSize = req.MinPagingSize } tasks = append(tasks, &copTask{ region: loc.Location.Region, @@ -868,6 +868,7 @@ func (worker *copIteratorWorker) handleCopPagingResult(bo *Backoffer, rpcCtx *ti // So we finish here. return nil, nil } + // calculate next ranges and grow the paging size task.ranges = worker.calculateRemain(task.ranges, pagingRange, worker.req.Desc) if task.ranges.Len() == 0 { diff --git a/store/copr/coprocessor_test.go b/store/copr/coprocessor_test.go index 7f2efa0e2db71..c32ce383374d1 100644 --- a/store/copr/coprocessor_test.go +++ b/store/copr/coprocessor_test.go @@ -509,6 +509,7 @@ func TestBuildPagingTasks(t *testing.T) { req := &kv.Request{} req.Paging = true + req.MinPagingSize = paging.MinPagingSize flashReq := &kv.Request{} flashReq.StoreType = kv.TiFlash tasks, err := buildCopTasks(bo, cache, buildCopRanges("a", "c"), req, nil) From 5eec739f219a876878c26b3d09fc0f4a109811f2 Mon Sep 17 00:00:00 2001 From: you06 Date: Wed, 13 Jul 2022 22:49:05 +0800 Subject: [PATCH 24/27] executor: optimize cursor read point get by reading through pessimistic lock cache (#36149) ref pingcap/tidb#36162 --- executor/point_get.go | 25 +++++++++++++++++++------ server/conn_test.go | 4 ++-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/executor/point_get.go b/executor/point_get.go index 04687403c0d4e..e744570cbf370 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -237,6 +237,17 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { return err } + // lockNonExistIdxKey indicates the key will be locked regardless of its existence. + lockNonExistIdxKey := !e.ctx.GetSessionVars().IsPessimisticReadConsistency() + // Non-exist keys are also locked if the isolation level is not read consistency, + // lock it before read here, then it's able to read from pessimistic lock cache. + if lockNonExistIdxKey { + err = e.lockKeyIfNeeded(ctx, e.idxKey) + if err != nil { + return err + } + } + e.handleVal, err = e.get(ctx, e.idxKey) if err != nil { if !kv.ErrNotExist.Equal(err) { @@ -244,12 +255,14 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { } } - // try lock the index key if isolation level is not read consistency // also lock key if read consistency read a value - if !e.ctx.GetSessionVars().IsPessimisticReadConsistency() || len(e.handleVal) > 0 { - err = e.lockKeyIfNeeded(ctx, e.idxKey) - if err != nil { - return err + // TODO: pessimistic lock support lock-if-exist. + if lockNonExistIdxKey || len(e.handleVal) > 0 { + if !lockNonExistIdxKey { + err = e.lockKeyIfNeeded(ctx, e.idxKey) + if err != nil { + return err + } } // Change the unique index LOCK into PUT record. if e.lock && len(e.handleVal) > 0 { @@ -377,7 +390,7 @@ func (e *PointGetExecutor) lockKeyIfNeeded(ctx context.Context, key []byte) erro return err } lockCtx.IterateValuesNotLocked(func(k, v []byte) { - seVars.TxnCtx.SetPessimisticLockCache(kv.Key(k), v) + seVars.TxnCtx.SetPessimisticLockCache(k, v) }) if len(e.handleVal) > 0 { seVars.TxnCtx.SetPessimisticLockCache(e.idxKey, e.handleVal) diff --git a/server/conn_test.go b/server/conn_test.go index f9661226ae1c3..df0d5fbef57d1 100644 --- a/server/conn_test.go +++ b/server/conn_test.go @@ -829,13 +829,13 @@ func TestPrefetchPointKeys(t *testing.T) { tk.MustExec("begin pessimistic") tk.MustExec("update prefetch set c = c + 1 where a = 2 and b = 2") - require.Equal(t, 1, tk.Session().GetSessionVars().TxnCtx.PessimisticCacheHit) + require.Equal(t, 2, tk.Session().GetSessionVars().TxnCtx.PessimisticCacheHit) err = cc.handleQuery(ctx, query) require.NoError(t, err) txn, err = tk.Session().Txn(false) require.NoError(t, err) require.True(t, txn.Valid()) - require.Equal(t, 5, tk.Session().GetSessionVars().TxnCtx.PessimisticCacheHit) + require.Equal(t, 6, tk.Session().GetSessionVars().TxnCtx.PessimisticCacheHit) tk.MustExec("commit") tk.MustQuery("select * from prefetch").Check(testkit.Rows("1 1 3", "2 2 6", "3 3 5")) } From 1885ebfaf0cce0721acb22dbcaefd5b922087020 Mon Sep 17 00:00:00 2001 From: Zak Zhao <57036248+joccau@users.noreply.github.com> Date: Wed, 13 Jul 2022 23:37:05 +0800 Subject: [PATCH 25/27] log-backup: get can restored global-checkpoint-ts when support v3 checkpoint advance (#36197) close pingcap/tidb#29501 --- br/pkg/task/stream.go | 6 ++++-- br/pkg/task/stream_test.go | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/br/pkg/task/stream.go b/br/pkg/task/stream.go index 160aa5ad6712a..c78351e9e0459 100644 --- a/br/pkg/task/stream.go +++ b/br/pkg/task/stream.go @@ -1339,9 +1339,11 @@ func getGlobalResolvedTS( return 0, errors.Trace(err) } var globalCheckpointTS uint64 = 0 - // TODO change the logic after checkpoint v3 implemented + // If V3 global-checkpoint advance, the maximum value in storeMap.resolvedTSMap as global-checkpoint-ts. + // If v2 global-checkpoint advance, it need the minimal value in storeMap.resolvedTSMap as global-checkpoint-ts. + // Because each of store maintains own checkpoint-ts only. for _, resolveTS := range storeMap.resolvedTSMap { - if resolveTS < globalCheckpointTS || globalCheckpointTS == 0 { + if globalCheckpointTS < resolveTS { globalCheckpointTS = resolveTS } } diff --git a/br/pkg/task/stream_test.go b/br/pkg/task/stream_test.go index abd113abfa711..e82f2eccb8cc3 100644 --- a/br/pkg/task/stream_test.go +++ b/br/pkg/task/stream_test.go @@ -277,7 +277,7 @@ func TestGetGlobalResolvedTS(t *testing.T) { require.Nil(t, err) globalResolvedTS, err := getGlobalResolvedTS(ctx, s) require.Nil(t, err) - require.Equal(t, uint64(100), globalResolvedTS) + require.Equal(t, uint64(101), globalResolvedTS) } func TestGetGlobalResolvedTS2(t *testing.T) { @@ -309,5 +309,5 @@ func TestGetGlobalResolvedTS2(t *testing.T) { require.Nil(t, err) globalResolvedTS, err := getGlobalResolvedTS(ctx, s) require.Nil(t, err) - require.Equal(t, uint64(98), globalResolvedTS) + require.Equal(t, uint64(99), globalResolvedTS) } From c6a212f0b5c268e9ca14304a684f75de3874042a Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Wed, 13 Jul 2022 23:57:05 +0800 Subject: [PATCH 26/27] store/copr: adjust the cop cache admission process time for paging (#36157) close pingcap/tidb#36156 --- store/copr/coprocessor.go | 2 +- store/copr/coprocessor_cache.go | 9 +++++++-- store/copr/coprocessor_cache_test.go | 22 +++++++++++----------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/store/copr/coprocessor.go b/store/copr/coprocessor.go index e7529fe6a5201..7e0eb74760233 100644 --- a/store/copr/coprocessor.go +++ b/store/copr/coprocessor.go @@ -992,7 +992,7 @@ func (worker *copIteratorWorker) handleCopResponse(bo *Backoffer, rpcCtx *tikv.R // Cache not hit or cache hit but not valid: update the cache if the response can be cached. if cacheKey != nil && resp.pbResp.CanBeCached && resp.pbResp.CacheLastVersion > 0 { if resp.detail != nil { - if worker.store.coprCache.CheckResponseAdmission(resp.pbResp.Data.Size(), resp.detail.TimeDetail.ProcessTime) { + if worker.store.coprCache.CheckResponseAdmission(resp.pbResp.Data.Size(), resp.detail.TimeDetail.ProcessTime, worker.req.Paging) { data := make([]byte, len(resp.pbResp.Data)) copy(data, resp.pbResp.Data) diff --git a/store/copr/coprocessor_cache.go b/store/copr/coprocessor_cache.go index 1adef9915cb48..2ee1ccffe2547 100644 --- a/store/copr/coprocessor_cache.go +++ b/store/copr/coprocessor_cache.go @@ -185,14 +185,19 @@ func (c *coprCache) CheckRequestAdmission(ranges int) bool { } // CheckResponseAdmission checks whether a response item is worth caching. -func (c *coprCache) CheckResponseAdmission(dataSize int, processTime time.Duration) bool { +func (c *coprCache) CheckResponseAdmission(dataSize int, processTime time.Duration, paging bool) bool { if c == nil { return false } if dataSize == 0 || dataSize > c.admissionMaxSize { return false } - if processTime < c.admissionMinProcessTime { + + admissionMinProcessTime := c.admissionMinProcessTime + if paging { + admissionMinProcessTime = admissionMinProcessTime / 3 + } + if processTime < admissionMinProcessTime { return false } return true diff --git a/store/copr/coprocessor_cache_test.go b/store/copr/coprocessor_cache_test.go index 91906c980d0f8..f629eb57f2c57 100644 --- a/store/copr/coprocessor_cache_test.go +++ b/store/copr/coprocessor_cache_test.go @@ -79,7 +79,7 @@ func TestDisable(t *testing.T) { v2 := cache.Get([]byte("foo")) require.Nil(t, v2) - v = cache.CheckResponseAdmission(1024, time.Second*5) + v = cache.CheckResponseAdmission(1024, time.Second*5, false) require.False(t, v) cache, err = newCoprCache(&config.CoprocessorCache{CapacityMB: 0.001}) @@ -104,34 +104,34 @@ func TestAdmission(t *testing.T) { v = cache.CheckRequestAdmission(1000) require.True(t, v) - v = cache.CheckResponseAdmission(0, 0) + v = cache.CheckResponseAdmission(0, 0, false) require.False(t, v) - v = cache.CheckResponseAdmission(0, 4*time.Millisecond) + v = cache.CheckResponseAdmission(0, 4*time.Millisecond, false) require.False(t, v) - v = cache.CheckResponseAdmission(0, 5*time.Millisecond) + v = cache.CheckResponseAdmission(0, 5*time.Millisecond, false) require.False(t, v) - v = cache.CheckResponseAdmission(1, 0) + v = cache.CheckResponseAdmission(1, 0, false) require.False(t, v) - v = cache.CheckResponseAdmission(1, 4*time.Millisecond) + v = cache.CheckResponseAdmission(1, 4*time.Millisecond, false) require.False(t, v) - v = cache.CheckResponseAdmission(1, 5*time.Millisecond) + v = cache.CheckResponseAdmission(1, 5*time.Millisecond, false) require.True(t, v) - v = cache.CheckResponseAdmission(1024, 5*time.Millisecond) + v = cache.CheckResponseAdmission(1024, 5*time.Millisecond, false) require.True(t, v) - v = cache.CheckResponseAdmission(1024*1024, 5*time.Millisecond) + v = cache.CheckResponseAdmission(1024*1024, 5*time.Millisecond, false) require.True(t, v) - v = cache.CheckResponseAdmission(1024*1024+1, 5*time.Millisecond) + v = cache.CheckResponseAdmission(1024*1024+1, 5*time.Millisecond, false) require.False(t, v) - v = cache.CheckResponseAdmission(1024*1024+1, 4*time.Millisecond) + v = cache.CheckResponseAdmission(1024*1024+1, 4*time.Millisecond, false) require.False(t, v) cache, err = newCoprCache(&config.CoprocessorCache{AdmissionMaxRanges: 5, AdmissionMinProcessMs: 5, AdmissionMaxResultMB: 1, CapacityMB: 1}) From 81cf12ebf651fa60d2bd1938ad22919faf3ba65e Mon Sep 17 00:00:00 2001 From: xufei Date: Thu, 14 Jul 2022 00:15:05 +0800 Subject: [PATCH 27/27] executor: parallel cancel mpp query (#36161) ref pingcap/tiflash#5095, close pingcap/tidb#36164 --- store/copr/mpp.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/store/copr/mpp.go b/store/copr/mpp.go index 22061039616a5..656fc11186ce1 100644 --- a/store/copr/mpp.go +++ b/store/copr/mpp.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/driver/backoff" derr "github.com/pingcap/tidb/store/driver/error" + "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/mathutil" "github.com/tikv/client-go/v2/tikv" @@ -341,13 +342,18 @@ func (m *mppIterator) cancelMppTasks() { } // send cancel cmd to all stores where tasks run + wg := util.WaitGroupWrapper{} for addr := range usedStoreAddrs { - _, err := m.store.GetTiKVClient().SendRequest(context.Background(), addr, wrappedReq, tikv.ReadTimeoutShort) - logutil.BgLogger().Debug("cancel task ", zap.Uint64("query id ", m.startTs), zap.String(" on addr ", addr)) - if err != nil { - logutil.BgLogger().Error("cancel task error: ", zap.Error(err), zap.Uint64(" for query id ", m.startTs), zap.String(" on addr ", addr)) - } + storeAddr := addr + wg.Run(func() { + _, err := m.store.GetTiKVClient().SendRequest(context.Background(), storeAddr, wrappedReq, tikv.ReadTimeoutShort) + logutil.BgLogger().Debug("cancel task", zap.Uint64("query id ", m.startTs), zap.String("on addr", storeAddr)) + if err != nil { + logutil.BgLogger().Error("cancel task error", zap.Error(err), zap.Uint64("query id", m.startTs), zap.String("on addr", storeAddr)) + } + }) } + wg.Wait() } func (m *mppIterator) establishMPPConns(bo *Backoffer, req *kv.MPPDispatchRequest, taskMeta *mpp.TaskMeta) {