diff --git a/DEPS.bzl b/DEPS.bzl index 621d3782f9714..b1806a24de0b7 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -4475,8 +4475,8 @@ def go_deps(): name = "org_golang_x_time", build_file_proto_mode = "disable_global", importpath = "golang.org/x/time", - sum = "h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE=", - version = "v0.2.0", + sum = "h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=", + version = "v0.3.0", ) go_repository( name = "org_golang_x_tools", diff --git a/bindinfo/handle.go b/bindinfo/handle.go index 907b6e94ccde0..59919e2b5ad85 100644 --- a/bindinfo/handle.go +++ b/bindinfo/handle.go @@ -533,6 +533,15 @@ func (h *BindHandle) SetBindRecordStatus(originalSQL string, binding *Binding, n return } +// SetBindRecordStatusByDigest set a BindRecord's status to the storage and bind cache. +func (h *BindHandle) SetBindRecordStatusByDigest(newStatus, sqlDigest string) (ok bool, err error) { + oldRecord, err := h.GetBindRecordBySQLDigest(sqlDigest) + if err != nil { + return false, err + } + return h.SetBindRecordStatus(oldRecord.OriginalSQL, nil, newStatus) +} + // GCBindRecord physically removes the deleted bind records in mysql.bind_info. func (h *BindHandle) GCBindRecord() (err error) { h.bindInfo.Lock() diff --git a/br/pkg/gluetidb/glue.go b/br/pkg/gluetidb/glue.go index 3b1675921afd3..19c3ef1c4c460 100644 --- a/br/pkg/gluetidb/glue.go +++ b/br/pkg/gluetidb/glue.go @@ -203,9 +203,32 @@ func (gs *tidbSession) CreatePlacementPolicy(ctx context.Context, policy *model. return d.CreatePlacementPolicyWithInfo(gs.se, policy, ddl.OnExistIgnore) } +// SplitBatchCreateTable provide a way to split batch into small batch when batch size is large than 6 MB. +// The raft entry has limit size of 6 MB, a batch of CreateTables may hit this limitation +// TODO: shall query string be set for each split batch create, it looks does not matter if we set once for all. +func (gs *tidbSession) SplitBatchCreateTable(schema model.CIStr, info []*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error { + var err error + d := domain.GetDomain(gs.se).DDL() + if err = d.BatchCreateTableWithInfo(gs.se, schema, info, append(cs, ddl.OnExistIgnore)...); kv.ErrEntryTooLarge.Equal(err) { + if len(info) == 1 { + return err + } + mid := len(info) / 2 + err = gs.SplitBatchCreateTable(schema, info[:mid]) + if err != nil { + return err + } + err = gs.SplitBatchCreateTable(schema, info[mid:]) + if err != nil { + return err + } + return nil + } + return err +} + // CreateTables implements glue.BatchCreateTableSession. func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error { - d := domain.GetDomain(gs.se).DDL() var dbName model.CIStr // Disable foreign key check when batch create tables. @@ -233,8 +256,7 @@ func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*mo cloneTables = append(cloneTables, table) } gs.se.SetValue(sessionctx.QueryString, queryBuilder.String()) - err := d.BatchCreateTableWithInfo(gs.se, dbName, cloneTables, append(cs, ddl.OnExistIgnore)...) - if err != nil { + if err := gs.SplitBatchCreateTable(dbName, cloneTables); err != nil { //It is possible to failure when TiDB does not support model.ActionCreateTables. //In this circumstance, BatchCreateTableWithInfo returns errno.ErrInvalidDDLJob, //we fall back to old way that creating table one by one diff --git a/br/pkg/lightning/backend/kv/sql2kv.go b/br/pkg/lightning/backend/kv/sql2kv.go index 6cebb1e29e329..9ad552ef5f340 100644 --- a/br/pkg/lightning/backend/kv/sql2kv.go +++ b/br/pkg/lightning/backend/kv/sql2kv.go @@ -169,7 +169,7 @@ func collectGeneratedColumns(se *session, meta *model.TableInfo, cols []*table.C var genCols []genCol for i, col := range cols { if col.GeneratedExpr != nil { - expr, err := expression.RewriteAstExpr(se, col.GeneratedExpr, schema, names) + expr, err := expression.RewriteAstExpr(se, col.GeneratedExpr, schema, names, false) if err != nil { return nil, err } diff --git a/br/pkg/lightning/restore/tidb.go b/br/pkg/lightning/restore/tidb.go index ddc8bb6c5ff19..00d49af646e3c 100644 --- a/br/pkg/lightning/restore/tidb.go +++ b/br/pkg/lightning/restore/tidb.go @@ -95,8 +95,10 @@ func DBFromConfig(ctx context.Context, dsn config.DBStore) (*sql.DB, error) { "tidb_opt_write_row_id": "1", // always set auto-commit to ON "autocommit": "1", - // alway set transaction mode to optimistic + // always set transaction mode to optimistic "tidb_txn_mode": "optimistic", + // disable foreign key checks + "foreign_key_checks": "0", } if dsn.Vars != nil { @@ -143,47 +145,6 @@ func (timgr *TiDBManager) Close() { timgr.db.Close() } -func InitSchema(ctx context.Context, g glue.Glue, database string, tablesSchema map[string]string) error { - logger := log.FromContext(ctx).With(zap.String("db", database)) - sqlExecutor := g.GetSQLExecutor() - - var createDatabase strings.Builder - createDatabase.WriteString("CREATE DATABASE IF NOT EXISTS ") - common.WriteMySQLIdentifier(&createDatabase, database) - err := sqlExecutor.ExecuteWithLog(ctx, createDatabase.String(), "create database", logger) - if err != nil { - return errors.Trace(err) - } - - task := logger.Begin(zap.InfoLevel, "create tables") - var sqlCreateStmts []string -loopCreate: - for tbl, sqlCreateTable := range tablesSchema { - task.Debug("create table", zap.String("schema", sqlCreateTable)) - - sqlCreateStmts, err = createIfNotExistsStmt(g.GetParser(), sqlCreateTable, database, tbl) - if err != nil { - break - } - - // TODO: maybe we should put these createStems into a transaction - for _, s := range sqlCreateStmts { - err = sqlExecutor.ExecuteWithLog( - ctx, - s, - "create table", - logger.With(zap.String("table", common.UniqueTable(database, tbl))), - ) - if err != nil { - break loopCreate - } - } - } - task.End(zap.ErrorLevel, err) - - return errors.Trace(err) -} - func createIfNotExistsStmt(p *parser.Parser, createTable, dbName, tblName string) ([]string, error) { stmts, _, err := p.ParseSQL(createTable) if err != nil { @@ -199,6 +160,9 @@ func createIfNotExistsStmt(p *parser.Parser, createTable, dbName, tblName string case *ast.CreateDatabaseStmt: node.Name = model.NewCIStr(dbName) node.IfNotExists = true + case *ast.DropDatabaseStmt: + node.Name = model.NewCIStr(dbName) + node.IfExists = true case *ast.CreateTableStmt: node.Table.Schema = model.NewCIStr(dbName) node.Table.Name = model.NewCIStr(tblName) diff --git a/br/pkg/lightning/restore/tidb_test.go b/br/pkg/lightning/restore/tidb_test.go index 9b204b2da22b1..a3710d822d2dd 100644 --- a/br/pkg/lightning/restore/tidb_test.go +++ b/br/pkg/lightning/restore/tidb_test.go @@ -165,97 +165,6 @@ func TestCreateTableIfNotExistsStmt(t *testing.T) { `, "m")) } -func TestInitSchema(t *testing.T) { - s := newTiDBSuite(t) - ctx := context.Background() - - s.mockDB. - ExpectExec("CREATE DATABASE IF NOT EXISTS `db`"). - WillReturnResult(sqlmock.NewResult(1, 1)) - s.mockDB. - ExpectExec("\\QCREATE TABLE IF NOT EXISTS `db`.`t1` (`a` INT PRIMARY KEY,`b` VARCHAR(200));\\E"). - WillReturnResult(sqlmock.NewResult(2, 1)) - s.mockDB. - ExpectExec("\\QSET @@SESSION.`FOREIGN_KEY_CHECKS`=0;\\E"). - WillReturnResult(sqlmock.NewResult(0, 0)) - s.mockDB. - ExpectExec("\\QCREATE TABLE IF NOT EXISTS `db`.`t2` (`xx` TEXT) AUTO_INCREMENT = 11203;\\E"). - WillReturnResult(sqlmock.NewResult(2, 1)) - s.mockDB. - ExpectClose() - - s.mockDB.MatchExpectationsInOrder(false) // maps are unordered. - err := InitSchema(ctx, s.tiGlue, "db", map[string]string{ - "t1": "create table t1 (a int primary key, b varchar(200));", - "t2": "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;CREATE TABLE `db`.`t2` (xx TEXT) AUTO_INCREMENT=11203;", - }) - s.mockDB.MatchExpectationsInOrder(true) - require.NoError(t, err) -} - -func TestInitSchemaSyntaxError(t *testing.T) { - s := newTiDBSuite(t) - ctx := context.Background() - - s.mockDB. - ExpectExec("CREATE DATABASE IF NOT EXISTS `db`"). - WillReturnResult(sqlmock.NewResult(1, 1)) - s.mockDB. - ExpectClose() - - err := InitSchema(ctx, s.tiGlue, "db", map[string]string{ - "t1": "create table `t1` with invalid syntax;", - }) - require.Error(t, err) -} - -func TestInitSchemaErrorLost(t *testing.T) { - s := newTiDBSuite(t) - ctx := context.Background() - - s.mockDB. - ExpectExec("CREATE DATABASE IF NOT EXISTS `db`"). - WillReturnResult(sqlmock.NewResult(1, 1)) - - s.mockDB. - ExpectExec("CREATE TABLE IF NOT EXISTS.*"). - WillReturnError(&mysql.MySQLError{ - Number: tmysql.ErrTooBigFieldlength, - Message: "Column length too big", - }) - - s.mockDB. - ExpectClose() - - err := InitSchema(ctx, s.tiGlue, "db", map[string]string{ - "t1": "create table `t1` (a int);", - "t2": "create table t2 (a int primary key, b varchar(200));", - }) - require.Regexp(t, ".*Column length too big.*", err.Error()) -} - -func TestInitSchemaUnsupportedSchemaError(t *testing.T) { - s := newTiDBSuite(t) - ctx := context.Background() - - s.mockDB. - ExpectExec("CREATE DATABASE IF NOT EXISTS `db`"). - WillReturnResult(sqlmock.NewResult(1, 1)) - s.mockDB. - ExpectExec("CREATE TABLE IF NOT EXISTS `db`.`t1`.*"). - WillReturnError(&mysql.MySQLError{ - Number: tmysql.ErrTooBigFieldlength, - Message: "Column length too big", - }) - s.mockDB. - ExpectClose() - - err := InitSchema(ctx, s.tiGlue, "db", map[string]string{ - "t1": "create table `t1` (a VARCHAR(999999999));", - }) - require.Regexp(t, ".*Column length too big.*", err.Error()) -} - func TestDropTable(t *testing.T) { s := newTiDBSuite(t) ctx := context.Background() diff --git a/br/tests/lightning_foreign_key/config.toml b/br/tests/lightning_foreign_key/config.toml new file mode 100644 index 0000000000000..3c85a830bfa22 --- /dev/null +++ b/br/tests/lightning_foreign_key/config.toml @@ -0,0 +1,3 @@ +[tikv-importer] +# Set on-duplicate=error to force using insert statement to write data. +on-duplicate = "error" diff --git a/br/tests/lightning_foreign_key/data/fk.t-schema.sql b/br/tests/lightning_foreign_key/data/fk.t-schema.sql new file mode 100644 index 0000000000000..98f00b9cadca8 --- /dev/null +++ b/br/tests/lightning_foreign_key/data/fk.t-schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE `t` +( + `a` bigint(20) NOT NULL, + `b` bigint(20) DEFAULT NULL, + PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */, + KEY `fk_1` (`b`), + CONSTRAINT `fk_1` FOREIGN KEY (`b`) REFERENCES `test`.`t2` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/br/tests/lightning_foreign_key/data/fk.t.csv b/br/tests/lightning_foreign_key/data/fk.t.csv new file mode 100644 index 0000000000000..cd0368580a4c8 --- /dev/null +++ b/br/tests/lightning_foreign_key/data/fk.t.csv @@ -0,0 +1,6 @@ +a,b +1,1 +2,2 +3,3 +4,4 +5,5 diff --git a/br/tests/lightning_foreign_key/run.sh b/br/tests/lightning_foreign_key/run.sh new file mode 100644 index 0000000000000..631b55d907102 --- /dev/null +++ b/br/tests/lightning_foreign_key/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright 2022 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +# Create existing tables that import data will reference. +run_sql 'CREATE DATABASE IF NOT EXISTS fk;' +run_sql 'CREATE TABLE fk.t2 (a BIGINT PRIMARY KEY);' + +for BACKEND in tidb local; do + run_sql 'DROP TABLE IF EXISTS fk.t;' + run_lightning --backend $BACKEND + run_sql 'SELECT GROUP_CONCAT(a) FROM fk.t ORDER BY a;' + check_contains '1,2,3,4,5' +done diff --git a/ddl/backfilling.go b/ddl/backfilling.go index d1035bad084bd..0f0910e1caf28 100644 --- a/ddl/backfilling.go +++ b/ddl/backfilling.go @@ -807,12 +807,7 @@ func (b *backfillScheduler) initCopReqSenderPool() { logutil.BgLogger().Warn("[ddl-ingest] cannot init cop request sender", zap.Error(err)) return } - ver, err := sessCtx.GetStore().CurrentVersion(kv.GlobalTxnScope) - if err != nil { - logutil.BgLogger().Warn("[ddl-ingest] cannot init cop request sender", zap.Error(err)) - return - } - b.copReqSenderPool = newCopReqSenderPool(b.ctx, copCtx, ver.Ver) + b.copReqSenderPool = newCopReqSenderPool(b.ctx, copCtx, sessCtx.GetStore()) } func (b *backfillScheduler) canSkipError(err error) bool { diff --git a/ddl/column_change_test.go b/ddl/column_change_test.go index 4528564d2f231..62623e1aca3c4 100644 --- a/ddl/column_change_test.go +++ b/ddl/column_change_test.go @@ -437,3 +437,27 @@ func testNewContext(store kv.Storage) sessionctx.Context { ctx.Store = store return ctx } + +func TestIssue40135(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + tk1 := testkit.NewTestKit(t, store) + tk1.MustExec("use test") + + tk.MustExec("CREATE TABLE t40135 ( a tinyint DEFAULT NULL, b varchar(32) DEFAULT 'md') PARTITION BY HASH (a) PARTITIONS 2") + one := true + hook := &ddl.TestDDLCallback{Do: dom} + var checkErr error + hook.OnJobRunBeforeExported = func(job *model.Job) { + if one { + one = false + _, checkErr = tk1.Exec("alter table t40135 change column a aNew SMALLINT NULL DEFAULT '-14996'") + } + } + dom.DDL().SetHook(hook) + tk.MustExec("alter table t40135 modify column a MEDIUMINT NULL DEFAULT '6243108' FIRST") + + require.ErrorContains(t, checkErr, "[ddl:8200]Unsupported modify column: Column 'a' has a partitioning function dependency and cannot be renamed") +} diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index 38aa43e4e7755..8d2777f26687c 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -2246,14 +2246,6 @@ func TestExchangePartitionTableCompatiable(t *testing.T) { "alter table pt8 exchange partition p0 with table nt8;", dbterror.ErrTablesDifferentMetadata, }, - { - // foreign key test - // Partition table doesn't support to add foreign keys in mysql - "create table pt9 (id int not null primary key auto_increment,t_id int not null) partition by hash(id) partitions 1;", - "create table nt9 (id int not null primary key auto_increment, t_id int not null,foreign key fk_id (t_id) references pt5(id));", - "alter table pt9 exchange partition p0 with table nt9;", - dbterror.ErrPartitionExchangeForeignKey, - }, { // Generated column (virtual) "create table pt10 (id int not null, lname varchar(30), fname varchar(100) generated always as (concat(lname,' ')) virtual) partition by hash(id) partitions 1;", @@ -4543,17 +4535,17 @@ func TestAlterModifyColumnOnPartitionedTableRename(t *testing.T) { tk.MustExec("create database " + schemaName) tk.MustExec("use " + schemaName) tk.MustExec(`create table t (a int, b char) partition by range (a) (partition p0 values less than (10))`) - tk.MustContainErrMsg(`alter table t change a c int`, "[planner:1054]Unknown column 'a' in 'expression'") + tk.MustContainErrMsg(`alter table t change a c int`, "[ddl:8200]Unsupported modify column: Column 'a' has a partitioning function dependency and cannot be renamed") tk.MustExec(`drop table t`) tk.MustExec(`create table t (a char, b char) partition by range columns (a) (partition p0 values less than ('z'))`) - tk.MustContainErrMsg(`alter table t change a c char`, "[ddl:8200]New column does not match partition definitions: [ddl:1567]partition column name cannot be found") + tk.MustContainErrMsg(`alter table t change a c char`, "[ddl:8200]Unsupported modify column: Column 'a' has a partitioning function dependency and cannot be renamed") tk.MustExec(`drop table t`) tk.MustExec(`create table t (a int, b char) partition by list (a) (partition p0 values in (10))`) - tk.MustContainErrMsg(`alter table t change a c int`, "[planner:1054]Unknown column 'a' in 'expression'") + tk.MustContainErrMsg(`alter table t change a c int`, "[ddl:8200]Unsupported modify column: Column 'a' has a partitioning function dependency and cannot be renamed") tk.MustExec(`drop table t`) tk.MustExec(`create table t (a char, b char) partition by list columns (a) (partition p0 values in ('z'))`) - tk.MustContainErrMsg(`alter table t change a c char`, "[ddl:8200]New column does not match partition definitions: [ddl:1567]partition column name cannot be found") + tk.MustContainErrMsg(`alter table t change a c char`, "[ddl:8200]Unsupported modify column: Column 'a' has a partitioning function dependency and cannot be renamed") tk.MustExec(`drop table t`) tk.MustExec(`create table t (a int, b char) partition by hash (a) partitions 3`) - tk.MustContainErrMsg(`alter table t change a c int`, "[planner:1054]Unknown column 'a' in 'expression'") + tk.MustContainErrMsg(`alter table t change a c int`, "[ddl:8200]Unsupported modify column: Column 'a' has a partitioning function dependency and cannot be renamed") } diff --git a/ddl/ddl.go b/ddl/ddl.go index 8c4d5235ea7ad..1e1b38eeb77bb 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -439,9 +439,9 @@ func (dc *ddlCtx) setDDLSourceForDiagnosis(job *model.Job) { ctx, exists := dc.jobCtx.jobCtxMap[job.ID] if !exists { ctx = NewJobContext() - ctx.setDDLLabelForDiagnosis(job) dc.jobCtx.jobCtxMap[job.ID] = ctx } + ctx.setDDLLabelForDiagnosis(job) } func (dc *ddlCtx) getResourceGroupTaggerForTopSQL(job *model.Job) tikvrpc.ResourceGroupTagger { @@ -1786,7 +1786,11 @@ func (s *session) execute(ctx context.Context, query string, label string) ([]ch defer func() { metrics.DDLJobTableDuration.WithLabelValues(label + "-" + metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) }() - rs, err := s.Context.(sqlexec.SQLExecutor).ExecuteInternal(kv.WithInternalSourceType(ctx, kv.InternalTxnDDL), query) + + if ctx.Value(kv.RequestSourceKey) == nil { + ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnDDL) + } + rs, err := s.Context.(sqlexec.SQLExecutor).ExecuteInternal(ctx, query) if err != nil { return nil, errors.Trace(err) } diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 2850a3aa968a5..3b609316eda85 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -1980,7 +1980,7 @@ func addIndexForForeignKey(ctx sessionctx.Context, tbInfo *model.TableInfo) erro if handleCol != nil && len(fk.Cols) == 1 && handleCol.Name.L == fk.Cols[0].L { continue } - if model.FindIndexByColumns(tbInfo, fk.Cols...) != nil { + if model.FindIndexByColumns(tbInfo, tbInfo.Indices, fk.Cols...) != nil { continue } idxName := fk.Name @@ -3264,6 +3264,14 @@ func (d *ddl) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast return dbterror.ErrOptOnCacheTable.GenWithStackByArgs("Alter Table") } } + // set name for anonymous foreign key. + maxForeignKeyID := tb.Meta().MaxForeignKeyID + for _, spec := range validSpecs { + if spec.Tp == ast.AlterTableAddConstraint && spec.Constraint.Tp == ast.ConstraintForeignKey && spec.Constraint.Name == "" { + maxForeignKeyID++ + spec.Constraint.Name = fmt.Sprintf("fk_%d", maxForeignKeyID) + } + } if len(validSpecs) > 1 { sctx.GetSessionVars().StmtCtx.MultiSchemaInfo = model.NewMultiSchemaInfo() @@ -4698,9 +4706,15 @@ func GetModifiableColumnJob( for _, name := range pt.GetPartitionColumnNames() { if strings.EqualFold(name.L, col.Name.L) { isPartitioningColumn = true + break } } if isPartitioningColumn { + // TODO: update the partitioning columns with new names if column is renamed + // Would be an extension from MySQL which does not support it. + if col.Name.L != newCol.Name.L { + return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs(fmt.Sprintf("Column '%s' has a partitioning function dependency and cannot be renamed", col.Name.O)) + } if !isColTypeAllowedAsPartitioningCol(newCol.FieldType) { return nil, dbterror.ErrNotAllowedTypeInPartition.GenWithStackByArgs(newCol.Name.O) } @@ -4744,7 +4758,6 @@ func GetModifiableColumnJob( newTblInfo.Columns = newCols var buf bytes.Buffer - // TODO: update the partitioning columns with new names if column is renamed AppendPartitionInfo(tblInfo.GetPartitionInfo(), &buf, mysql.ModeNone) // The parser supports ALTER TABLE ... PARTITION BY ... even if the ddl code does not yet :) // Ignoring warnings @@ -6164,7 +6177,7 @@ func (d *ddl) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexName m // After DDL job is put to the queue, and if the check fail, TiDB will run the DDL cancel logic. // The recover step causes DDL wait a few seconds, makes the unit test painfully slow. // For same reason, decide whether index is global here. - indexColumns, err := buildIndexColumns(ctx, tblInfo.Columns, indexPartSpecifications) + indexColumns, _, err := buildIndexColumns(ctx, tblInfo.Columns, indexPartSpecifications) if err != nil { return errors.Trace(err) } @@ -6274,7 +6287,7 @@ func BuildHiddenColumnInfo(ctx sessionctx.Context, indexPartSpecifications []*as if err != nil { return nil, errors.Trace(err) } - expr, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, idxPart.Expr) + expr, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, idxPart.Expr, true) if err != nil { // TODO: refine the error message. return nil, err @@ -6389,7 +6402,7 @@ func (d *ddl) createIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.Inde // After DDL job is put to the queue, and if the check fail, TiDB will run the DDL cancel logic. // The recover step causes DDL wait a few seconds, makes the unit test painfully slow. // For same reason, decide whether index is global here. - indexColumns, err := buildIndexColumns(ctx, finalColumns, indexPartSpecifications) + indexColumns, _, err := buildIndexColumns(ctx, finalColumns, indexPartSpecifications) if err != nil { return errors.Trace(err) } @@ -6570,7 +6583,7 @@ func (d *ddl) CreateForeignKey(ctx sessionctx.Context, ti ast.Ident, fkName mode if err != nil { return err } - if model.FindIndexByColumns(t.Meta(), fkInfo.Cols...) == nil { + if model.FindIndexByColumns(t.Meta(), t.Meta().Indices, fkInfo.Cols...) == nil { // Need to auto create index for fk cols if ctx.GetSessionVars().StmtCtx.MultiSchemaInfo == nil { ctx.GetSessionVars().StmtCtx.MultiSchemaInfo = model.NewMultiSchemaInfo() diff --git a/ddl/ddl_worker.go b/ddl/ddl_worker.go index 8621dcb08361c..89f466b1497c5 100644 --- a/ddl/ddl_worker.go +++ b/ddl/ddl_worker.go @@ -119,7 +119,7 @@ func NewJobContext() *JobContext { cacheSQL: "", cacheNormalizedSQL: "", cacheDigest: nil, - tp: "unknown", + tp: "", } } @@ -536,16 +536,20 @@ func cleanMDLInfo(pool *sessionPool, jobID int64, ec *clientv3.Client) { } // checkMDLInfo checks if metadata lock info exists. It means the schema is locked by some TiDBs if exists. -func checkMDLInfo(jobID int64, pool *sessionPool) (bool, error) { - sql := fmt.Sprintf("select * from mysql.tidb_mdl_info where job_id = %d", jobID) +func checkMDLInfo(jobID int64, pool *sessionPool) (bool, int64, error) { + sql := fmt.Sprintf("select version from mysql.tidb_mdl_info where job_id = %d", jobID) sctx, _ := pool.get() defer pool.put(sctx) sess := newSession(sctx) rows, err := sess.execute(context.Background(), sql, "check-mdl-info") if err != nil { - return false, err + return false, 0, err } - return len(rows) > 0, nil + if len(rows) == 0 { + return false, 0, nil + } + ver := rows[0].GetInt64(0) + return true, ver, nil } func needUpdateRawArgs(job *model.Job, meetErr bool) bool { @@ -757,6 +761,9 @@ func getDDLRequestSource(job *model.Job) string { } func (w *JobContext) setDDLLabelForDiagnosis(job *model.Job) { + if w.tp != "" { + return + } w.tp = getDDLRequestSource(job) w.ddlJobCtx = kv.WithInternalSourceType(w.ddlJobCtx, w.ddlJobSourceType()) } @@ -1377,6 +1384,32 @@ func waitSchemaChanged(ctx context.Context, d *ddlCtx, waitTime time.Duration, l zap.String("job", job.String())) } +// waitSchemaSyncedForMDL likes waitSchemaSynced, but it waits for getting the metadata lock of the latest version of this DDL. +func waitSchemaSyncedForMDL(d *ddlCtx, job *model.Job, latestSchemaVersion int64) error { + failpoint.Inject("checkDownBeforeUpdateGlobalVersion", func(val failpoint.Value) { + if val.(bool) { + if mockDDLErrOnce > 0 && mockDDLErrOnce != latestSchemaVersion { + panic("check down before update global version failed") + } else { + mockDDLErrOnce = -1 + } + } + }) + + timeStart := time.Now() + // OwnerCheckAllVersions returns only when all TiDB schemas are synced(exclude the isolated TiDB). + err := d.schemaSyncer.OwnerCheckAllVersions(context.Background(), job.ID, latestSchemaVersion) + if err != nil { + logutil.Logger(d.ctx).Info("[ddl] wait latest schema version encounter error", zap.Int64("ver", latestSchemaVersion), zap.Error(err)) + return err + } + logutil.Logger(d.ctx).Info("[ddl] wait latest schema version changed(get the metadata lock if tidb_enable_metadata_lock is true)", + zap.Int64("ver", latestSchemaVersion), + zap.Duration("take time", time.Since(timeStart)), + zap.String("job", job.String())) + return nil +} + // waitSchemaSynced handles the following situation: // If the job enters a new state, and the worker crashs when it's in the process of waiting for 2 * lease time, // Then the worker restarts quickly, we may run the job immediately again, diff --git a/ddl/export_test.go b/ddl/export_test.go index 486390f9a6810..3ea26fb04290c 100644 --- a/ddl/export_test.go +++ b/ddl/export_test.go @@ -28,7 +28,7 @@ func SetBatchInsertDeleteRangeSize(i int) { var NewCopContext4Test = newCopContext -func FetchRowsFromCop4Test(copCtx *copContext, startKey, endKey kv.Key, startTS uint64, +func FetchRowsFromCop4Test(copCtx *copContext, startKey, endKey kv.Key, store kv.Storage, batchSize int) ([]*indexRecord, bool, error) { variable.SetDDLReorgBatchSize(int32(batchSize)) task := &reorgBackfillTask{ @@ -36,7 +36,7 @@ func FetchRowsFromCop4Test(copCtx *copContext, startKey, endKey kv.Key, startTS startKey: startKey, endKey: endKey, } - pool := newCopReqSenderPool(context.Background(), copCtx, startTS) + pool := newCopReqSenderPool(context.Background(), copCtx, store) pool.adjustSize(1) pool.tasksCh <- task idxRec, _, _, done, err := pool.fetchRowColValsFromCop(*task) diff --git a/ddl/fktest/foreign_key_test.go b/ddl/fktest/foreign_key_test.go index 911f86f49ec85..ba466a8cef07b 100644 --- a/ddl/fktest/foreign_key_test.go +++ b/ddl/fktest/foreign_key_test.go @@ -19,6 +19,7 @@ import ( "context" "fmt" "testing" + "time" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/infoschema" @@ -319,6 +320,24 @@ func TestCreateTableWithForeignKeyPrivilegeCheck(t *testing.T) { tk2.MustExec("create table t4 (a int, foreign key fk(a) references t1(id), foreign key (a) references t3(id));") } +func TestAlterTableWithForeignKeyPrivilegeCheck(t *testing.T) { + store, _ := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create user 'u1'@'%' identified by '';") + tk.MustExec("grant create,alter on *.* to 'u1'@'%';") + tk.MustExec("create table t1 (id int key);") + tk2 := testkit.NewTestKit(t, store) + tk2.MustExec("use test") + tk2.Session().Auth(&auth.UserIdentity{Username: "u1", Hostname: "localhost", CurrentUser: true, AuthUsername: "u1", AuthHostname: "%"}, nil, []byte("012345678901234567890")) + tk2.MustExec("create table t2 (a int)") + err := tk2.ExecToErr("alter table t2 add foreign key (a) references t1 (id) on update cascade") + require.Error(t, err) + require.Equal(t, "[planner:1142]REFERENCES command denied to user 'u1'@'%' for table 't1'", err.Error()) + tk.MustExec("grant references on test.t1 to 'u1'@'%';") + tk2.MustExec("alter table t2 add foreign key (a) references t1 (id) on update cascade") +} + func TestRenameTableWithForeignKeyMetaInfo(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) @@ -648,6 +667,28 @@ func TestCreateTableWithForeignKeyError(t *testing.T) { create: "create table t2 (id int key, constraint fk foreign key (id) references t1(name5678901234567890123456789012345678901234567890123456789012345));", err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long", }, + // Test foreign key with temporary table + { + refer: "create temporary table t1 (id int key);", + create: "create table t2 (id int key, constraint fk foreign key (id) references t1(id));", + err: "[schema:1824]Failed to open the referenced table 't1'", + }, + { + refer: "create table t1 (id int key);", + create: "create temporary table t2 (id int key, constraint fk foreign key (id) references t1(id));", + err: "[schema:1215]Cannot add foreign key constraint", + }, + // Test foreign key with partition table + { + refer: "create table t1 (id int key) partition by hash(id) partitions 3;", + create: "create table t2 (id int key, constraint fk foreign key (id) references t1(id));", + err: "[schema:1506]Foreign key clause is not yet supported in conjunction with partitioning", + }, + { + refer: "create table t1 (id int key);", + create: "create table t2 (id int key, constraint fk foreign key (id) references t1(id)) partition by hash(id) partitions 3;", + err: "[schema:1506]Foreign key clause is not yet supported in conjunction with partitioning", + }, } for _, ca := range cases { tk.MustExec("drop table if exists t2") @@ -1396,6 +1437,40 @@ func TestAlterTableAddForeignKeyError(t *testing.T) { alter: "alter table t2 add constraint name5678901234567890123456789012345678901234567890123456789012345 foreign key (b) references t1(id)", err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long", }, + // Test foreign key with temporary table. + { + prepares: []string{ + "create temporary table t1 (id int key);", + "create table t2 (a int, b int unique);", + }, + alter: "alter table t2 add constraint fk foreign key (b) references t1(id)", + err: "[schema:1824]Failed to open the referenced table 't1'", + }, + { + prepares: []string{ + "create table t1 (id int key);", + "create temporary table t2 (a int, b int unique);", + }, + alter: "alter table t2 add constraint fk foreign key (b) references t1(id)", + err: "[ddl:8200]TiDB doesn't support ALTER TABLE for local temporary table", + }, + // Test foreign key with partition table + { + prepares: []string{ + "create table t1 (id int key) partition by hash(id) partitions 3;", + "create table t2 (id int key);", + }, + alter: "alter table t2 add constraint fk foreign key (id) references t1(id)", + err: "[schema:1506]Foreign key clause is not yet supported in conjunction with partitioning", + }, + { + prepares: []string{ + "create table t1 (id int key);", + "create table t2 (id int key) partition by hash(id) partitions 3;;", + }, + alter: "alter table t2 add constraint fk foreign key (id) references t1(id)", + err: "[schema:1506]Foreign key clause is not yet supported in conjunction with partitioning", + }, } for i, ca := range cases { tk.MustExec("drop table if exists t2") @@ -1544,3 +1619,84 @@ func getLatestSchemaDiff(t *testing.T, tk *testkit.TestKit) *model.SchemaDiff { require.NoError(t, err) return diff } + +func TestMultiSchemaAddForeignKey(t *testing.T) { + store, _ := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@foreign_key_checks=1;") + tk.MustExec("use test") + tk.MustExec("create table t1 (id int key);") + tk.MustExec("create table t2 (a int, b int);") + tk.MustExec("alter table t2 add foreign key (a) references t1(id), add foreign key (b) references t1(id)") + tk.MustExec("alter table t2 add column c int, add column d int") + tk.MustExec("alter table t2 add foreign key (c) references t1(id), add foreign key (d) references t1(id), add index(c), add index(d)") + tk.MustExec("drop table t2") + tk.MustExec("create table t2 (a int, b int, index idx1(a), index idx2(b));") + tk.MustGetErrMsg("alter table t2 drop index idx1, drop index idx2, add foreign key (a) references t1(id), add foreign key (b) references t1(id)", + "[ddl:1553]Cannot drop index 'idx1': needed in a foreign key constraint") + tk.MustExec("alter table t2 drop index idx1, drop index idx2") + tk.MustExec("alter table t2 add foreign key (a) references t1(id), add foreign key (b) references t1(id)") + tk.MustQuery("show create table t2").Check(testkit.Rows("t2 CREATE TABLE `t2` (\n" + + " `a` int(11) DEFAULT NULL,\n" + + " `b` int(11) DEFAULT NULL,\n" + + " KEY `fk_1` (`a`),\n" + + " KEY `fk_2` (`b`),\n" + + " CONSTRAINT `fk_1` FOREIGN KEY (`a`) REFERENCES `test`.`t1` (`id`),\n" + + " CONSTRAINT `fk_2` FOREIGN KEY (`b`) REFERENCES `test`.`t1` (`id`)\n" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")) + tk.MustExec("drop table t2") + tk.MustExec("create table t2 (a int, b int, index idx0(a,b), index idx1(a), index idx2(b));") + tk.MustExec("alter table t2 drop index idx1, add foreign key (a) references t1(id), add foreign key (b) references t1(id)") +} + +func TestAddForeignKeyInBigTable(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@foreign_key_checks=1;") + tk.MustExec("use test") + tk.MustExec("create table employee (id bigint auto_increment key, pid bigint)") + tk.MustExec("insert into employee (id) values (1),(2),(3),(4),(5),(6),(7),(8)") + for i := 0; i < 14; i++ { + tk.MustExec("insert into employee (pid) select pid from employee") + } + tk.MustExec("update employee set pid=id-1 where id>1") + start := time.Now() + tk.MustExec("alter table employee add foreign key fk_1(pid) references employee(id)") + require.Less(t, time.Since(start), time.Minute) +} + +func TestForeignKeyWithCacheTable(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@foreign_key_checks=1;") + tk.MustExec("use test") + // Test foreign key refer cache table. + tk.MustExec("create table t1 (id int key);") + tk.MustExec("insert into t1 values (1),(2),(3),(4)") + tk.MustExec("alter table t1 cache;") + tk.MustExec("create table t2 (b int);") + tk.MustExec("alter table t2 add constraint fk foreign key (b) references t1(id) on delete cascade on update cascade") + tk.MustExec("insert into t2 values (1),(2),(3),(4)") + tk.MustGetDBError("insert into t2 values (5)", plannercore.ErrNoReferencedRow2) + tk.MustExec("update t1 set id = id+10 where id=1") + tk.MustExec("delete from t1 where id<10") + tk.MustQuery("select * from t1").Check(testkit.Rows("11")) + tk.MustQuery("select * from t2").Check(testkit.Rows("11")) + tk.MustExec("alter table t1 nocache;") + tk.MustExec("drop table t1,t2;") + + // Test add foreign key on cache table. + tk.MustExec("create table t1 (id int key);") + tk.MustExec("create table t2 (b int);") + tk.MustExec("alter table t2 add constraint fk foreign key (b) references t1(id) on delete cascade on update cascade") + tk.MustExec("alter table t2 cache;") + tk.MustExec("insert into t1 values (1),(2),(3),(4)") + tk.MustExec("insert into t2 values (1),(2),(3),(4)") + tk.MustGetDBError("insert into t2 values (5)", plannercore.ErrNoReferencedRow2) + tk.MustExec("update t1 set id = id+10 where id=1") + tk.MustExec("delete from t1 where id<10") + tk.MustQuery("select * from t1").Check(testkit.Rows("11")) + tk.MustQuery("select * from t2").Check(testkit.Rows("11")) + tk.MustExec("alter table t2 nocache;") + tk.MustExec("drop table t1,t2;") +} diff --git a/ddl/foreign_key.go b/ddl/foreign_key.go index af9570fbee837..1a06719cb404b 100644 --- a/ddl/foreign_key.go +++ b/ddl/foreign_key.go @@ -266,6 +266,9 @@ func checkTableForeignKey(referTblInfo, tblInfo *model.TableInfo, fkInfo *model. if referTblInfo.TTLInfo != nil { return dbterror.ErrUnsupportedTTLReferencedByFK } + if referTblInfo.GetPartitionInfo() != nil || tblInfo.GetPartitionInfo() != nil { + return infoschema.ErrForeignKeyOnPartitioned + } // check refer columns in parent table. for i := range fkInfo.RefCols { @@ -291,7 +294,7 @@ func checkTableForeignKey(referTblInfo, tblInfo *model.TableInfo, fkInfo *model. } } // check refer columns should have index. - if model.FindIndexByColumns(referTblInfo, fkInfo.RefCols...) == nil { + if model.FindIndexByColumns(referTblInfo, referTblInfo.Indices, fkInfo.RefCols...) == nil { return infoschema.ErrForeignKeyNoIndexInParent.GenWithStackByArgs(fkInfo.Name, fkInfo.RefTable) } return nil @@ -660,7 +663,7 @@ func checkAddForeignKeyValidInOwner(d *ddlCtx, t *meta.Meta, schema string, tbIn return nil } } - if model.FindIndexByColumns(tbInfo, fk.Cols...) == nil { + if model.FindIndexByColumns(tbInfo, tbInfo.Indices, fk.Cols...) == nil { return errors.Errorf("Failed to add the foreign key constraint. Missing index for '%s' foreign key columns in the table '%s'", fk.Name, tbInfo.Name) } return nil @@ -674,7 +677,12 @@ func checkForeignKeyConstrain(w *worker, schema, table string, fkInfo *model.FKI if err != nil { return errors.Trace(err) } - defer w.sessPool.put(sctx) + originValue := sctx.GetSessionVars().OptimizerEnableNAAJ + sctx.GetSessionVars().OptimizerEnableNAAJ = true + defer func() { + sctx.GetSessionVars().OptimizerEnableNAAJ = originValue + w.sessPool.put(sctx) + }() var buf strings.Builder buf.WriteString("select 1 from %n.%n where ") @@ -709,7 +717,7 @@ func checkForeignKeyConstrain(w *worker, schema, table string, fkInfo *model.FKI } buf.WriteString(" from %n.%n ) limit 1") paramsList = append(paramsList, fkInfo.RefSchema.L, fkInfo.RefTable.L) - rows, _, err := sctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(w.ctx, nil, buf.String(), paramsList...) + rows, _, err := sctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(w.ctx, []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseCurSession}, buf.String(), paramsList...) if err != nil { return errors.Trace(err) } diff --git a/ddl/generated_column.go b/ddl/generated_column.go index 2f4ceee8b60a9..678d803edf521 100644 --- a/ddl/generated_column.go +++ b/ddl/generated_column.go @@ -268,12 +268,14 @@ func checkModifyGeneratedColumn(sctx sessionctx.Context, tbl table.Table, oldCol } type illegalFunctionChecker struct { - hasIllegalFunc bool - hasAggFunc bool - hasRowVal bool // hasRowVal checks whether the functional index refers to a row value - hasWindowFunc bool - hasNotGAFunc4ExprIdx bool - otherErr error + hasIllegalFunc bool + hasAggFunc bool + hasRowVal bool // hasRowVal checks whether the functional index refers to a row value + hasWindowFunc bool + hasNotGAFunc4ExprIdx bool + hasCastArrayFunc bool + disallowCastArrayFunc bool + otherErr error } func (c *illegalFunctionChecker) Enter(inNode ast.Node) (outNode ast.Node, skipChildren bool) { @@ -308,7 +310,14 @@ func (c *illegalFunctionChecker) Enter(inNode ast.Node) (outNode ast.Node, skipC case *ast.WindowFuncExpr: c.hasWindowFunc = true return inNode, true + case *ast.FuncCastExpr: + c.hasCastArrayFunc = c.hasCastArrayFunc || node.Tp.IsArray() + if c.disallowCastArrayFunc && node.Tp.IsArray() { + c.otherErr = expression.ErrNotSupportedYet.GenWithStackByArgs("Use of CAST( .. AS .. ARRAY) outside of functional index in CREATE(non-SELECT)/ALTER TABLE or in general expressions") + return inNode, true + } } + c.disallowCastArrayFunc = true return inNode, false } @@ -355,6 +364,9 @@ func checkIllegalFn4Generated(name string, genType int, expr ast.ExprNode) error if genType == typeIndex && c.hasNotGAFunc4ExprIdx && !config.GetGlobalConfig().Experimental.AllowsExpressionIndex { return dbterror.ErrUnsupportedExpressionIndex } + if genType == typeColumn && c.hasCastArrayFunc { + return expression.ErrNotSupportedYet.GenWithStackByArgs("Use of CAST( .. AS .. ARRAY) outside of functional index in CREATE(non-SELECT)/ALTER TABLE or in general expressions") + } return nil } diff --git a/ddl/index.go b/ddl/index.go index 0f70b73b61046..7c9fc1d083ff4 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -64,26 +64,33 @@ var ( telemetryAddIndexIngestUsage = metrics.TelemetryAddIndexIngestCnt ) -func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, indexPartSpecifications []*ast.IndexPartSpecification) ([]*model.IndexColumn, error) { +func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, indexPartSpecifications []*ast.IndexPartSpecification) ([]*model.IndexColumn, bool, error) { // Build offsets. idxParts := make([]*model.IndexColumn, 0, len(indexPartSpecifications)) var col *model.ColumnInfo + var mvIndex bool maxIndexLength := config.GetGlobalConfig().MaxIndexLength // The sum of length of all index columns. sumLength := 0 for _, ip := range indexPartSpecifications { col = model.FindColumnInfo(columns, ip.Column.Name.L) if col == nil { - return nil, dbterror.ErrKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ip.Column.Name) + return nil, false, dbterror.ErrKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ip.Column.Name) } if err := checkIndexColumn(ctx, col, ip.Length); err != nil { - return nil, err + return nil, false, err + } + if col.FieldType.IsArray() { + if mvIndex { + return nil, false, dbterror.ErrNotSupportedYet.GenWithStack("'more than one multi-valued key part per index'") + } + mvIndex = true } indexColLen := ip.Length indexColumnLength, err := getIndexColumnLength(col, ip.Length) if err != nil { - return nil, err + return nil, false, err } sumLength += indexColumnLength @@ -92,12 +99,12 @@ func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, inde // The multiple column index and the unique index in which the length sum exceeds the maximum size // will return an error instead produce a warning. if ctx == nil || ctx.GetSessionVars().StrictSQLMode || mysql.HasUniKeyFlag(col.GetFlag()) || len(indexPartSpecifications) > 1 { - return nil, dbterror.ErrTooLongKey.GenWithStackByArgs(maxIndexLength) + return nil, false, dbterror.ErrTooLongKey.GenWithStackByArgs(maxIndexLength) } // truncate index length and produce warning message in non-restrict sql mode. colLenPerUint, err := getIndexColumnLength(col, 1) if err != nil { - return nil, err + return nil, false, err } indexColLen = maxIndexLength / colLenPerUint // produce warning message @@ -111,7 +118,7 @@ func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, inde }) } - return idxParts, nil + return idxParts, mvIndex, nil } // CheckPKOnGeneratedColumn checks the specification of PK is valid. @@ -154,7 +161,7 @@ func checkIndexColumn(ctx sessionctx.Context, col *model.ColumnInfo, indexColumn } // JSON column cannot index. - if col.FieldType.GetType() == mysql.TypeJSON { + if col.FieldType.GetType() == mysql.TypeJSON && !col.FieldType.IsArray() { if col.Hidden { return dbterror.ErrFunctionalIndexOnJSONOrGeometryFunction } @@ -263,7 +270,7 @@ func BuildIndexInfo( return nil, errors.Trace(err) } - idxColumns, err := buildIndexColumns(ctx, allTableColumns, indexPartSpecifications) + idxColumns, mvIndex, err := buildIndexColumns(ctx, allTableColumns, indexPartSpecifications) if err != nil { return nil, errors.Trace(err) } @@ -276,6 +283,7 @@ func BuildIndexInfo( Primary: isPrimary, Unique: isUnique, Global: isGlobal, + MVIndex: mvIndex, } if indexOption != nil { diff --git a/ddl/index_cop.go b/ddl/index_cop.go index 0a04ac63eb190..fab097727139b 100644 --- a/ddl/index_cop.go +++ b/ddl/index_cop.go @@ -103,9 +103,9 @@ type copReqSenderPool struct { resultsCh chan idxRecResult results generic.SyncMap[int, struct{}] - ctx context.Context - copCtx *copContext - startTS uint64 + ctx context.Context + copCtx *copContext + store kv.Storage senders []*copReqSender wg sync.WaitGroup @@ -139,7 +139,12 @@ func (c *copReqSender) run() { curTaskID = task.id logutil.BgLogger().Info("[ddl-ingest] start a cop-request task", zap.Int("id", task.id), zap.String("task", task.String())) - rs, err := p.copCtx.buildTableScan(p.ctx, p.startTS, task.startKey, task.excludedEndKey()) + ver, err := p.store.CurrentVersion(kv.GlobalTxnScope) + if err != nil { + p.resultsCh <- idxRecResult{id: task.id, err: err} + return + } + rs, err := p.copCtx.buildTableScan(p.ctx, ver.Ver, task.startKey, task.excludedEndKey()) if err != nil { p.resultsCh <- idxRecResult{id: task.id, err: err} return @@ -167,7 +172,7 @@ func (c *copReqSender) run() { } } -func newCopReqSenderPool(ctx context.Context, copCtx *copContext, startTS uint64) *copReqSenderPool { +func newCopReqSenderPool(ctx context.Context, copCtx *copContext, store kv.Storage) *copReqSenderPool { poolSize := copReadChunkPoolSize() idxBufPool := make(chan []*indexRecord, poolSize) srcChkPool := make(chan *chunk.Chunk, poolSize) @@ -181,7 +186,7 @@ func newCopReqSenderPool(ctx context.Context, copCtx *copContext, startTS uint64 results: generic.NewSyncMap[int, struct{}](10), ctx: ctx, copCtx: copCtx, - startTS: startTS, + store: store, senders: make([]*copReqSender, 0, variable.GetDDLReorgWorkerCounter()), wg: sync.WaitGroup{}, idxBufPool: idxBufPool, diff --git a/ddl/index_cop_test.go b/ddl/index_cop_test.go index 80e37f6a74121..38bced0b6678d 100644 --- a/ddl/index_cop_test.go +++ b/ddl/index_cop_test.go @@ -43,7 +43,7 @@ func TestAddIndexFetchRowsFromCoprocessor(t *testing.T) { endKey := startKey.PrefixNext() txn, err := store.Begin() require.NoError(t, err) - idxRec, done, err := ddl.FetchRowsFromCop4Test(copCtx, startKey, endKey, txn.StartTS(), 10) + idxRec, done, err := ddl.FetchRowsFromCop4Test(copCtx, startKey, endKey, store, 10) require.NoError(t, err) require.False(t, done) require.NoError(t, txn.Rollback()) diff --git a/ddl/job_table.go b/ddl/job_table.go index a6e19b7f7edf0..117b3722eccde 100644 --- a/ddl/job_table.go +++ b/ddl/job_table.go @@ -237,7 +237,7 @@ func (d *ddl) delivery2worker(wk *worker, pool *workerPool, job *model.Job) { // check if this ddl job is synced to all servers. if !d.isSynced(job) || d.once.Load() { if variable.EnableMDL.Load() { - exist, err := checkMDLInfo(job.ID, d.sessPool) + exist, version, err := checkMDLInfo(job.ID, d.sessPool) if err != nil { logutil.BgLogger().Warn("[ddl] check MDL info failed", zap.Error(err), zap.String("job", job.String())) // Release the worker resource. @@ -246,10 +246,8 @@ func (d *ddl) delivery2worker(wk *worker, pool *workerPool, job *model.Job) { } else if exist { // Release the worker resource. pool.put(wk) - err = waitSchemaSynced(d.ddlCtx, job, 2*d.lease) + err = waitSchemaSyncedForMDL(d.ddlCtx, job, version) if err != nil { - logutil.BgLogger().Warn("[ddl] wait ddl job sync failed", zap.Error(err), zap.String("job", job.String())) - time.Sleep(time.Second) return } d.once.Store(false) @@ -398,7 +396,8 @@ func updateDDLJob2Table(sctx *session, job *model.Job, updateRawArgs bool) error // getDDLReorgHandle gets DDL reorg handle. func getDDLReorgHandle(sess *session, job *model.Job) (element *meta.Element, startKey, endKey kv.Key, physicalTableID int64, err error) { sql := fmt.Sprintf("select ele_id, ele_type, start_key, end_key, physical_id from mysql.tidb_ddl_reorg where job_id = %d", job.ID) - rows, err := sess.execute(context.Background(), sql, "get_handle") + ctx := kv.WithInternalSourceType(context.Background(), getDDLRequestSource(job)) + rows, err := sess.execute(ctx, sql, "get_handle") if err != nil { return nil, nil, nil, 0, err } diff --git a/ddl/multi_schema_change.go b/ddl/multi_schema_change.go index ab306fe546932..a21bc27965c77 100644 --- a/ddl/multi_schema_change.go +++ b/ddl/multi_schema_change.go @@ -261,7 +261,10 @@ func fillMultiSchemaInfo(info *model.MultiSchemaInfo, job *model.Job) (err error case model.ActionRebaseAutoID, model.ActionModifyTableComment, model.ActionModifyTableCharsetAndCollate: case model.ActionAddForeignKey: fkInfo := job.Args[0].(*model.FKInfo) - info.ForeignKeys = append(info.ForeignKeys, fkInfo.Name) + info.AddForeignKeys = append(info.AddForeignKeys, model.AddForeignKeyInfo{ + Name: fkInfo.Name, + Cols: fkInfo.Cols, + }) default: return dbterror.ErrRunMultiSchemaChanges.FastGenByArgs(job.Type.String()) } @@ -323,6 +326,32 @@ func checkOperateSameColAndIdx(info *model.MultiSchemaInfo) error { return checkIndexes(info.AlterIndexes, true) } +func checkOperateDropIndexUseByForeignKey(info *model.MultiSchemaInfo, t table.Table) error { + var remainIndexes, droppingIndexes []*model.IndexInfo + tbInfo := t.Meta() + for _, idx := range tbInfo.Indices { + dropping := false + for _, name := range info.DropIndexes { + if name.L == idx.Name.L { + dropping = true + break + } + } + if dropping { + droppingIndexes = append(droppingIndexes, idx) + } else { + remainIndexes = append(remainIndexes, idx) + } + } + + for _, fk := range info.AddForeignKeys { + if droppingIdx := model.FindIndexByColumns(tbInfo, droppingIndexes, fk.Cols...); droppingIdx != nil && model.FindIndexByColumns(tbInfo, remainIndexes, fk.Cols...) == nil { + return dbterror.ErrDropIndexNeededInForeignKey.GenWithStackByArgs(droppingIdx.Name) + } + } + return nil +} + func checkMultiSchemaInfo(info *model.MultiSchemaInfo, t table.Table) error { err := checkOperateSameColAndIdx(info) if err != nil { @@ -334,6 +363,11 @@ func checkMultiSchemaInfo(info *model.MultiSchemaInfo, t table.Table) error { return err } + err = checkOperateDropIndexUseByForeignKey(info, t) + if err != nil { + return err + } + return checkAddColumnTooManyColumns(len(t.Cols()) + len(info.AddColumns) - len(info.DropColumns)) } diff --git a/ddl/partition.go b/ddl/partition.go index 0a1ea4e6fbe66..2c95f389707f9 100644 --- a/ddl/partition.go +++ b/ddl/partition.go @@ -1375,7 +1375,7 @@ func checkPartitionFuncType(ctx sessionctx.Context, expr ast.ExprNode, tblInfo * return nil } - e, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, expr) + e, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, expr, false) if err != nil { return errors.Trace(err) } diff --git a/docs/design/2020-08-04-global-index.md b/docs/design/2020-08-04-global-index.md index 80078688777b7..f5e2d89f932c4 100644 --- a/docs/design/2020-08-04-global-index.md +++ b/docs/design/2020-08-04-global-index.md @@ -183,7 +183,7 @@ In TiDB, operators in the partitioned table will be translated to UnionAll in th ## Compatibility -MySQL does not support global index, which means this feature may cause some compatibility issues. We add an option `enable_global_index` in `config.Config` to control it. The default value of this option is `false`, so TiDB will keep consistent with MySQL, unless the user open global index feature manually. +MySQL does not support global index, which means this feature may cause some compatibility issues. We add an option `enable-global-index` in `config.Config` to control it. The default value of this option is `false`, so TiDB will keep consistent with MySQL, unless the user open global index feature manually. ## Implementation diff --git a/domain/plan_replayer.go b/domain/plan_replayer.go index 4df05a627f3e2..c0f3231223c74 100644 --- a/domain/plan_replayer.go +++ b/domain/plan_replayer.go @@ -384,7 +384,7 @@ func (w *planReplayerTaskDumpWorker) HandleTask(task *PlanReplayerDumpTask) (suc return true } - file, fileName, err := replayer.GeneratePlanReplayerFile() + file, fileName, err := replayer.GeneratePlanReplayerFile(task.IsContinuesCapture) if err != nil { logutil.BgLogger().Warn("generate plan replayer capture task file failed", zap.String("sqlDigest", taskKey.SQLDigest), diff --git a/errors.toml b/errors.toml index 32b8d40e902b2..0c5f5765d1269 100644 --- a/errors.toml +++ b/errors.toml @@ -766,6 +766,11 @@ error = ''' Incorrect usage of %s and %s ''' +["ddl:1235"] +error = ''' +This version of TiDB doesn't yet support '%s' +''' + ["ddl:1246"] error = ''' Converting column '%s' from %s to %s @@ -2406,6 +2411,11 @@ error = ''' Changing schema from '%-.192s' to '%-.192s' is not allowed. ''' +["schema:1506"] +error = ''' +Foreign key clause is not yet supported in conjunction with partitioning +''' + ["schema:1822"] error = ''' Failed to add the foreign key constraint. Missing index for constraint '%s' in the referenced table '%s' diff --git a/executor/admin_test.go b/executor/admin_test.go index 41b926cf2c377..0b2530e76d5a3 100644 --- a/executor/admin_test.go +++ b/executor/admin_test.go @@ -1095,9 +1095,7 @@ func TestCheckFailReport(t *testing.T) { require.NoError(t, txn.Commit(tk.ctx)) ctx, hook := withLogHook(tk.ctx, t, "inconsistency") - _, err = tk.Exec(ctx, "admin check table admin_test") - require.Error(t, err) - require.Equal(t, "[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 1, index-values:\"\" != record-values:\"handle: 1, values: [KindInt64 1]\"", err.Error()) + tk.MustGetErrMsg(ctx, "admin check table admin_test", "[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 1, index-values:\"\" != record-values:\"handle: 1, values: [KindInt64 1]\"") hook.checkLogCount(t, 1) hook.logs[0].checkMsg(t, "admin check found data inconsistency") hook.logs[0].checkField(t, @@ -1119,9 +1117,7 @@ func TestCheckFailReport(t *testing.T) { require.NoError(t, txn.Commit(tk.ctx)) ctx, hook := withLogHook(tk.ctx, t, "inconsistency") - _, err = tk.Exec(ctx, "admin check table admin_test") - require.Error(t, err) - require.Equal(t, "[admin:8223]data inconsistency in table: admin_test, index: k2, handle: 1, index-values:\"\" != record-values:\"handle: 1, values: [KindString 10]\"", err.Error()) + tk.MustGetErrMsg(ctx, "admin check table admin_test", "[admin:8223]data inconsistency in table: admin_test, index: k2, handle: 1, index-values:\"\" != record-values:\"handle: 1, values: [KindString 10]\"") hook.checkLogCount(t, 1) hook.logs[0].checkMsg(t, "admin check found data inconsistency") hook.logs[0].checkField(t, @@ -1143,9 +1139,8 @@ func TestCheckFailReport(t *testing.T) { require.NoError(t, txn.Commit(tk.ctx)) ctx, hook := withLogHook(tk.ctx, t, "inconsistency") - _, err = tk.Exec(ctx, "admin check table admin_test") - require.Error(t, err) - require.Equal(t, "[admin:8223]data inconsistency in table: admin_test, index: k2, handle: 1, index-values:\"handle: 1, values: [KindString 100 KindInt64 1]\" != record-values:\"\"", err.Error()) + tk.MustGetErrMsg(ctx, "admin check table admin_test", + "[admin:8223]data inconsistency in table: admin_test, index: k2, handle: 1, index-values:\"handle: 1, values: [KindString 100 KindInt64 1]\" != record-values:\"\"") hook.checkLogCount(t, 1) logEntry := hook.logs[0] logEntry.checkMsg(t, "admin check found data inconsistency") @@ -1188,9 +1183,8 @@ func TestCheckFailReport(t *testing.T) { require.NoError(t, txn.Commit(tk.ctx)) ctx, hook := withLogHook(tk.ctx, t, "inconsistency") - _, err = tk.Exec(ctx, "admin check table admin_test") - require.Error(t, err) - require.Equal(t, "[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 1, index-values:\"handle: 1, values: [KindInt64 10 KindInt64 1]\" != record-values:\"\"", err.Error()) + tk.MustGetErrMsg(ctx, "admin check table admin_test", + "[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 1, index-values:\"handle: 1, values: [KindInt64 10 KindInt64 1]\" != record-values:\"\"") hook.checkLogCount(t, 1) logEntry := hook.logs[0] logEntry.checkMsg(t, "admin check found data inconsistency") @@ -1233,9 +1227,8 @@ func TestCheckFailReport(t *testing.T) { require.NoError(t, err) require.NoError(t, txn.Commit(tk.ctx)) ctx, hook := withLogHook(tk.ctx, t, "inconsistency") - _, err = tk.Exec(ctx, "admin check table admin_test") - require.Error(t, err) - require.Equal(t, "[executor:8134]data inconsistency in table: admin_test, index: uk1, col: c2, handle: \"1\", index-values:\"KindInt64 20\" != record-values:\"KindInt64 10\", compare err:", err.Error()) + tk.MustGetErrMsg(ctx, "admin check table admin_test", + "[executor:8134]data inconsistency in table: admin_test, index: uk1, col: c2, handle: \"1\", index-values:\"KindInt64 20\" != record-values:\"KindInt64 10\", compare err:") hook.checkLogCount(t, 1) logEntry := hook.logs[0] logEntry.checkMsg(t, "admin check found data inconsistency") @@ -1261,9 +1254,8 @@ func TestCheckFailReport(t *testing.T) { require.NoError(t, err) require.NoError(t, txn.Commit(tk.ctx)) ctx, hook := withLogHook(tk.ctx, t, "inconsistency") - _, err = tk.Exec(ctx, "admin check table admin_test") - require.Error(t, err) - require.Equal(t, "[executor:8134]data inconsistency in table: admin_test, index: k2, col: c3, handle: \"1\", index-values:\"KindString 200\" != record-values:\"KindString 100\", compare err:", err.Error()) + tk.MustGetErrMsg(ctx, "admin check table admin_test", + "[executor:8134]data inconsistency in table: admin_test, index: k2, col: c3, handle: \"1\", index-values:\"KindString 200\" != record-values:\"KindString 100\", compare err:") hook.checkLogCount(t, 1) logEntry := hook.logs[0] logEntry.checkMsg(t, "admin check found data inconsistency") @@ -1301,12 +1293,10 @@ func TestCheckFailReport(t *testing.T) { // TODO(tiancaiamao): admin check doesn't support the chunk protocol. // Remove this after https://github.com/pingcap/tidb/issues/35156 - _, err = tk.Exec(ctx, "set @@tidb_enable_chunk_rpc = off") - require.NoError(t, err) + tk.MustExec(ctx, "set @@tidb_enable_chunk_rpc = off") - _, err = tk.Exec(ctx, "admin check table admin_test") - require.Error(t, err) - require.Equal(t, `[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 282574488403969, index-values:"handle: 282574488403969, values: [KindInt64 282578800083201 KindInt64 282574488403969]" != record-values:""`, err.Error()) + tk.MustGetErrMsg(ctx, "admin check table admin_test", + `[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 282574488403969, index-values:"handle: 282574488403969, values: [KindInt64 282578800083201 KindInt64 282574488403969]" != record-values:""`) hook.checkLogCount(t, 1) logEntry := hook.logs[0] logEntry.checkMsg(t, "admin check found data inconsistency") diff --git a/executor/analyzetest/BUILD.bazel b/executor/analyzetest/BUILD.bazel index 53126213363a5..3112abe57c00f 100644 --- a/executor/analyzetest/BUILD.bazel +++ b/executor/analyzetest/BUILD.bazel @@ -8,7 +8,6 @@ go_test( "main_test.go", ], flaky = True, - race = "on", shard_count = 50, deps = [ "//domain", @@ -30,6 +29,7 @@ go_test( "//tablecodec", "//testkit", "//types", + "//util", "//util/codec", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", diff --git a/executor/analyzetest/analyze_test.go b/executor/analyzetest/analyze_test.go index e3bf9d51d9260..55f3ad9397be9 100644 --- a/executor/analyzetest/analyze_test.go +++ b/executor/analyzetest/analyze_test.go @@ -17,6 +17,7 @@ package analyzetest import ( "context" "fmt" + "runtime" "strconv" "strings" "testing" @@ -43,6 +44,7 @@ import ( "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/codec" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/testutils" @@ -3060,3 +3062,115 @@ func TestAutoAnalyzeAwareGlobalVariableChange(t *testing.T) { require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/injectBaseCount")) require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/injectBaseModifyCount")) } + +func TestGlobalMemoryControlForAnalyze(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + + tk0 := testkit.NewTestKit(t, store) + tk0.MustExec("set global tidb_mem_oom_action = 'cancel'") + tk0.MustExec("set global tidb_server_memory_limit = 512MB") + tk0.MustExec("set global tidb_server_memory_limit_sess_min_size = 128") + + sm := &testkit.MockSessionManager{ + PS: []*util.ProcessInfo{tk0.Session().ShowProcess()}, + } + dom.ServerMemoryLimitHandle().SetSessionManager(sm) + go dom.ServerMemoryLimitHandle().Run() + + tk0.MustExec("use test") + tk0.MustExec("create table t(a int)") + tk0.MustExec("insert into t select 1") + for i := 1; i <= 8; i++ { + tk0.MustExec("insert into t select * from t") // 256 Lines + } + sql := "analyze table t with 1.0 samplerate;" // Need about 100MB + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/util/memory/ReadMemStats", `return(536870912)`)) + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/mockAnalyzeMergeWorkerSlowConsume", `return(100)`)) + defer func() { + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/util/memory/ReadMemStats")) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/mockAnalyzeMergeWorkerSlowConsume")) + }() + _, err := tk0.Exec(sql) + require.True(t, strings.Contains(err.Error(), "Out Of Memory Quota!")) + runtime.GC() +} + +func TestGlobalMemoryControlForAutoAnalyze(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + originalVal1 := tk.MustQuery("select @@global.tidb_mem_oom_action").Rows()[0][0].(string) + tk.MustExec("set global tidb_mem_oom_action = 'cancel'") + //originalVal2 := tk.MustQuery("select @@global.tidb_server_memory_limit").Rows()[0][0].(string) + tk.MustExec("set global tidb_server_memory_limit = 512MB") + originalVal3 := tk.MustQuery("select @@global.tidb_server_memory_limit_sess_min_size").Rows()[0][0].(string) + tk.MustExec("set global tidb_server_memory_limit_sess_min_size = 128") + defer func() { + tk.MustExec(fmt.Sprintf("set global tidb_mem_oom_action = %v", originalVal1)) + //tk.MustExec(fmt.Sprintf("set global tidb_server_memory_limit = %v", originalVal2)) + tk.MustExec(fmt.Sprintf("set global tidb_server_memory_limit_sess_min_size = %v", originalVal3)) + }() + + // clean child trackers + oldChildTrackers := executor.GlobalAnalyzeMemoryTracker.GetChildrenForTest() + for _, tracker := range oldChildTrackers { + tracker.Detach() + } + defer func() { + for _, tracker := range oldChildTrackers { + tracker.AttachTo(executor.GlobalAnalyzeMemoryTracker) + } + }() + childTrackers := executor.GlobalAnalyzeMemoryTracker.GetChildrenForTest() + require.Len(t, childTrackers, 0) + + tk.MustExec("use test") + tk.MustExec("create table t(a int)") + tk.MustExec("insert into t select 1") + for i := 1; i <= 8; i++ { + tk.MustExec("insert into t select * from t") // 256 Lines + } + _, err0 := tk.Exec("analyze table t with 1.0 samplerate;") + require.NoError(t, err0) + rs0 := tk.MustQuery("select fail_reason from mysql.analyze_jobs where table_name=? and state=? limit 1", "t", "failed") + require.Len(t, rs0.Rows(), 0) + + h := dom.StatsHandle() + originalVal4 := handle.AutoAnalyzeMinCnt + originalVal5 := tk.MustQuery("select @@global.tidb_auto_analyze_ratio").Rows()[0][0].(string) + handle.AutoAnalyzeMinCnt = 0 + tk.MustExec("set global tidb_auto_analyze_ratio = 0.001") + defer func() { + handle.AutoAnalyzeMinCnt = originalVal4 + tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal5)) + }() + + sm := &testkit.MockSessionManager{ + Dom: dom, + PS: []*util.ProcessInfo{tk.Session().ShowProcess()}, + } + dom.ServerMemoryLimitHandle().SetSessionManager(sm) + go dom.ServerMemoryLimitHandle().Run() + + tk.MustExec("insert into t values(4),(5),(6)") + require.NoError(t, h.DumpStatsDeltaToKV(handle.DumpAll)) + err := h.Update(dom.InfoSchema()) + require.NoError(t, err) + + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/util/memory/ReadMemStats", `return(536870912)`)) + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/mockAnalyzeMergeWorkerSlowConsume", `return(100)`)) + defer func() { + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/util/memory/ReadMemStats")) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/mockAnalyzeMergeWorkerSlowConsume")) + }() + tk.MustQuery("select 1") + childTrackers = executor.GlobalAnalyzeMemoryTracker.GetChildrenForTest() + require.Len(t, childTrackers, 0) + + h.HandleAutoAnalyze(dom.InfoSchema()) + rs := tk.MustQuery("select fail_reason from mysql.analyze_jobs where table_name=? and state=? limit 1", "t", "failed") + failReason := rs.Rows()[0][0].(string) + require.True(t, strings.Contains(failReason, "Out Of Memory Quota!")) + + childTrackers = executor.GlobalAnalyzeMemoryTracker.GetChildrenForTest() + require.Len(t, childTrackers, 0) +} diff --git a/executor/bind.go b/executor/bind.go index c5f254121a9bf..90272e6878620 100644 --- a/executor/bind.go +++ b/executor/bind.go @@ -63,6 +63,8 @@ func (e *SQLBindExec) Next(ctx context.Context, req *chunk.Chunk) error { return e.reloadBindings() case plannercore.OpSetBindingStatus: return e.setBindingStatus() + case plannercore.OpSetBindingStatusByDigest: + return e.setBindingStatusByDigest() default: return errors.Errorf("unsupported SQL bind operation: %v", e.sqlBindOp) } @@ -119,6 +121,15 @@ func (e *SQLBindExec) setBindingStatus() error { return err } +func (e *SQLBindExec) setBindingStatusByDigest() error { + ok, err := domain.GetDomain(e.ctx).BindHandle().SetBindRecordStatusByDigest(e.newStatus, e.sqlDigest) + if err == nil && !ok { + warningMess := errors.New("There are no bindings can be set the status. Please check the SQL text") + e.ctx.GetSessionVars().StmtCtx.AppendWarning(warningMess) + } + return err +} + func (e *SQLBindExec) createSQLBind() error { // For audit log, SQLBindExec execute "explain" statement internally, save and recover stmtctx // is necessary to avoid 'create binding' been recorded as 'explain'. diff --git a/executor/builder.go b/executor/builder.go index b33c57d3de234..d4270397eecd0 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -3511,17 +3511,39 @@ func buildIndexRangeForEachPartition(ctx sessionctx.Context, usedPartitions []ta return nextRange, nil } -func keyColumnsIncludeAllPartitionColumns(keyColumns []int, pe *tables.PartitionExpr) bool { - tmp := make(map[int]struct{}, len(keyColumns)) - for _, offset := range keyColumns { - tmp[offset] = struct{}{} +func getPartitionKeyColOffsets(keyColIDs []int64, pt table.PartitionedTable) []int { + keyColOffsets := make([]int, len(keyColIDs)) + for i, colID := range keyColIDs { + offset := -1 + for j, col := range pt.Cols() { + if colID == col.ID { + offset = j + break + } + } + if offset == -1 { + return nil + } + keyColOffsets[i] = offset + } + + pe, err := pt.(interface { + PartitionExpr() (*tables.PartitionExpr, error) + }).PartitionExpr() + if err != nil { + return nil + } + + offsetMap := make(map[int]struct{}) + for _, offset := range keyColOffsets { + offsetMap[offset] = struct{}{} } for _, offset := range pe.ColumnOffset { - if _, ok := tmp[offset]; !ok { - return false + if _, ok := offsetMap[offset]; !ok { + return nil } } - return true + return keyColOffsets } func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table, schema *expression.Schema, partitionInfo *plannercore.PartitionInfo, @@ -3536,15 +3558,6 @@ func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table return nil, false, nil, err } - // check whether can runtime prune. - type partitionExpr interface { - PartitionExpr() (*tables.PartitionExpr, error) - } - pe, err := tbl.(partitionExpr).PartitionExpr() - if err != nil { - return nil, false, nil, err - } - // recalculate key column offsets if len(lookUpContent) == 0 { return nil, false, nil, nil @@ -3552,29 +3565,9 @@ func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table if lookUpContent[0].keyColIDs == nil { return nil, false, nil, plannercore.ErrInternal.GenWithStack("cannot get column IDs when dynamic pruning") } - keyColOffsets := make([]int, len(lookUpContent[0].keyColIDs)) - for i, colID := range lookUpContent[0].keyColIDs { - offset := -1 - for j, col := range partitionTbl.Cols() { - if colID == col.ID { - offset = j - break - } - } - if offset == -1 { - return nil, false, nil, plannercore.ErrInternal.GenWithStack("invalid column offset when dynamic pruning") - } - keyColOffsets[i] = offset - } - - offsetMap := make(map[int]bool) - for _, offset := range keyColOffsets { - offsetMap[offset] = true - } - for _, offset := range pe.ColumnOffset { - if _, ok := offsetMap[offset]; !ok { - return condPruneResult, false, nil, nil - } + keyColOffsets := getPartitionKeyColOffsets(lookUpContent[0].keyColIDs, partitionTbl) + if len(keyColOffsets) == 0 { + return condPruneResult, false, nil, nil } locateKey := make([]types.Datum, len(partitionTbl.Cols())) @@ -4149,12 +4142,6 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte } tbl, _ := builder.is.TableByID(tbInfo.ID) pt := tbl.(table.PartitionedTable) - pe, err := tbl.(interface { - PartitionExpr() (*tables.PartitionExpr, error) - }).PartitionExpr() - if err != nil { - return nil, err - } partitionInfo := &v.PartitionInfo usedPartitionList, err := builder.partitionPruning(pt, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames) if err != nil { @@ -4165,8 +4152,12 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte usedPartitions[p.GetPhysicalID()] = p } var kvRanges []kv.KeyRange + var keyColOffsets []int + if len(lookUpContents) > 0 { + keyColOffsets = getPartitionKeyColOffsets(lookUpContents[0].keyColIDs, pt) + } if v.IsCommonHandle { - if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) { + if len(keyColOffsets) > 0 { locateKey := make([]types.Datum, e.Schema().Len()) kvRanges = make([]kv.KeyRange, 0, len(lookUpContents)) // lookUpContentsByPID groups lookUpContents by pid(partition) so that kv ranges for same partition can be merged. @@ -4212,7 +4203,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte handles, lookUpContents := dedupHandles(lookUpContents) - if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) { + if len(keyColOffsets) > 0 { locateKey := make([]types.Datum, e.Schema().Len()) kvRanges = make([]kv.KeyRange, 0, len(lookUpContents)) for _, content := range lookUpContents { diff --git a/executor/compiler.go b/executor/compiler.go index cfe71f2ccdf29..e2c2a29794d1d 100644 --- a/executor/compiler.go +++ b/executor/compiler.go @@ -158,14 +158,16 @@ func (c *Compiler) Compile(ctx context.Context, stmtNode ast.StmtNode) (_ *ExecS } } if c.Ctx.GetSessionVars().IsPlanReplayerCaptureEnabled() && !c.Ctx.GetSessionVars().InRestrictedSQL { - startTS, err := sessiontxn.GetTxnManager(c.Ctx).GetStmtReadTS() - if err != nil { - return nil, err - } - if c.Ctx.GetSessionVars().EnablePlanReplayedContinuesCapture { - checkPlanReplayerContinuesCapture(c.Ctx, stmtNode, startTS) - } else { - checkPlanReplayerCaptureTask(c.Ctx, stmtNode, startTS) + if _, ok := stmtNode.(*ast.SelectStmt); ok { + startTS, err := sessiontxn.GetTxnManager(c.Ctx).GetStmtReadTS() + if err != nil { + return nil, err + } + if c.Ctx.GetSessionVars().EnablePlanReplayedContinuesCapture { + checkPlanReplayerContinuesCapture(c.Ctx, stmtNode, startTS) + } else { + checkPlanReplayerCaptureTask(c.Ctx, stmtNode, startTS) + } } } diff --git a/executor/delete.go b/executor/delete.go index 3aa0932a07c22..97b3487ffa3f9 100644 --- a/executor/delete.go +++ b/executor/delete.go @@ -245,7 +245,8 @@ func (e *DeleteExec) removeRow(ctx sessionctx.Context, t table.Table, h kv.Handl if err != nil { return err } - err = e.onRemoveRowForFK(ctx, t, data) + tid := t.Meta().ID + err = onRemoveRowForFK(ctx, data, e.fkChecks[tid], e.fkCascades[tid]) if err != nil { return err } @@ -253,8 +254,7 @@ func (e *DeleteExec) removeRow(ctx sessionctx.Context, t table.Table, h kv.Handl return nil } -func (e *DeleteExec) onRemoveRowForFK(ctx sessionctx.Context, t table.Table, data []types.Datum) error { - fkChecks := e.fkChecks[t.Meta().ID] +func onRemoveRowForFK(ctx sessionctx.Context, data []types.Datum, fkChecks []*FKCheckExec, fkCascades []*FKCascadeExec) error { sc := ctx.GetSessionVars().StmtCtx for _, fkc := range fkChecks { err := fkc.deleteRowNeedToCheck(sc, data) @@ -262,7 +262,6 @@ func (e *DeleteExec) onRemoveRowForFK(ctx sessionctx.Context, t table.Table, dat return err } } - fkCascades := e.fkCascades[t.Meta().ID] for _, fkc := range fkCascades { err := fkc.onDeleteRow(sc, data) if err != nil { diff --git a/executor/executor.go b/executor/executor.go index 90622ce52e527..9f95e63aaed20 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1933,7 +1933,7 @@ func (e *UnionExec) Close() error { func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { vars := ctx.GetSessionVars() var sc *stmtctx.StatementContext - if vars.TxnCtx.CouldRetry { + if vars.TxnCtx.CouldRetry || mysql.HasCursorExistsFlag(vars.Status) { // Must construct new statement context object, the retry history need context for every statement. // TODO: Maybe one day we can get rid of transaction retry, then this logic can be deleted. sc = &stmtctx.StatementContext{} @@ -1965,6 +1965,7 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { sc.SysdateIsNow = ctx.GetSessionVars().SysdateIsNow + vars.MemTracker.Detach() vars.MemTracker.UnbindActions() vars.MemTracker.SetBytesLimit(vars.MemQuotaQuery) vars.MemTracker.ResetMaxConsumed() diff --git a/executor/executor_test.go b/executor/executor_test.go index bd64c39e5a134..122fbdbe7dd2f 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -6199,38 +6199,6 @@ func TestGlobalMemoryControl2(t *testing.T) { runtime.GC() } -func TestGlobalMemoryControlForAnalyze(t *testing.T) { - store, dom := testkit.CreateMockStoreAndDomain(t) - - tk0 := testkit.NewTestKit(t, store) - tk0.MustExec("set global tidb_mem_oom_action = 'cancel'") - tk0.MustExec("set global tidb_server_memory_limit = 512MB") - tk0.MustExec("set global tidb_server_memory_limit_sess_min_size = 128") - - sm := &testkit.MockSessionManager{ - PS: []*util.ProcessInfo{tk0.Session().ShowProcess()}, - } - dom.ServerMemoryLimitHandle().SetSessionManager(sm) - go dom.ServerMemoryLimitHandle().Run() - - tk0.MustExec("use test") - tk0.MustExec("create table t(a int)") - tk0.MustExec("insert into t select 1") - for i := 1; i <= 8; i++ { - tk0.MustExec("insert into t select * from t") // 256 Lines - } - sql := "analyze table t with 1.0 samplerate;" // Need about 100MB - require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/util/memory/ReadMemStats", `return(536870912)`)) - require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/mockAnalyzeMergeWorkerSlowConsume", `return(100)`)) - defer func() { - require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/util/memory/ReadMemStats")) - require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/mockAnalyzeMergeWorkerSlowConsume")) - }() - _, err := tk0.Exec(sql) - require.True(t, strings.Contains(err.Error(), "Out Of Memory Quota!")) - runtime.GC() -} - func TestCompileOutOfMemoryQuota(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/executor/fktest/foreign_key_test.go b/executor/fktest/foreign_key_test.go index a162bd22b96aa..8d6442f39fad4 100644 --- a/executor/fktest/foreign_key_test.go +++ b/executor/fktest/foreign_key_test.go @@ -2539,3 +2539,107 @@ func TestForeignKeyIssue39732(t *testing.T) { tk.MustExec("execute stmt1 using @a;") tk.MustQuery("select * from t1 order by id").Check(testkit.Rows()) } + +func TestForeignKeyOnReplaceIntoChildTable(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@global.tidb_enable_foreign_key=1") + tk.MustExec("set @@foreign_key_checks=1") + tk.MustExec("use test") + tk.MustExec("create table t_data (id int, a int, b int)") + tk.MustExec("insert into t_data (id, a, b) values (1, 1, 1), (2, 2, 2);") + for _, ca := range foreignKeyTestCase1 { + tk.MustExec("drop table if exists t2;") + tk.MustExec("drop table if exists t1;") + for _, sql := range ca.prepareSQLs { + tk.MustExec(sql) + } + tk.MustExec("replace into t1 (id, a, b) values (1, 1, 1);") + tk.MustExec("replace into t2 (id, a, b) values (1, 1, 1)") + tk.MustGetDBError("replace into t1 (id, a, b) values (1, 2, 3);", plannercore.ErrRowIsReferenced2) + if !ca.notNull { + tk.MustExec("replace into t2 (id, a, b) values (2, null, 1)") + tk.MustExec("replace into t2 (id, a, b) values (3, 1, null)") + tk.MustExec("replace into t2 (id, a, b) values (4, null, null)") + } + tk.MustGetDBError("replace into t2 (id, a, b) values (5, 1, 0);", plannercore.ErrNoReferencedRow2) + tk.MustGetDBError("replace into t2 (id, a, b) values (6, 0, 1);", plannercore.ErrNoReferencedRow2) + tk.MustGetDBError("replace into t2 (id, a, b) values (7, 2, 2);", plannercore.ErrNoReferencedRow2) + // Test replace into from select. + tk.MustExec("delete from t2") + tk.MustExec("replace into t2 (id, a, b) select id, a, b from t_data where t_data.id=1") + tk.MustGetDBError("replace into t2 (id, a, b) select id, a, b from t_data where t_data.id=2", plannercore.ErrNoReferencedRow2) + + // Test in txn + tk.MustExec("delete from t2") + tk.MustExec("begin") + tk.MustExec("delete from t1 where a=1") + tk.MustGetDBError("replace into t2 (id, a, b) values (1, 1, 1)", plannercore.ErrNoReferencedRow2) + tk.MustExec("replace into t1 (id, a, b) values (2, 2, 2)") + tk.MustExec("replace into t2 (id, a, b) values (2, 2, 2)") + tk.MustGetDBError("replace into t1 (id, a, b) values (2, 2, 3);", plannercore.ErrRowIsReferenced2) + tk.MustExec("rollback") + tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("1 1 1")) + tk.MustQuery("select id, a, b from t2 order by id").Check(testkit.Rows()) + } + + // Case-10: test primary key is handle and contain foreign key column, and foreign key column has default value. + tk.MustExec("drop table if exists t2;") + tk.MustExec("drop table if exists t1;") + tk.MustExec("set @@tidb_enable_clustered_index=0;") + tk.MustExec("create table t1 (id int,a int, primary key(id));") + tk.MustExec("create table t2 (id int key,a int not null default 0, index (a), foreign key fk(a) references t1(id));") + tk.MustExec("replace into t1 values (1, 1);") + tk.MustExec("replace into t2 values (1, 1);") + tk.MustGetDBError("replace into t2 (id) values (10);", plannercore.ErrNoReferencedRow2) + tk.MustGetDBError("replace into t2 values (3, 2);", plannercore.ErrNoReferencedRow2) + + // Case-11: test primary key is handle and contain foreign key column, and foreign key column doesn't have default value. + tk.MustExec("drop table if exists t2;") + tk.MustExec("create table t2 (id int key,a int, index (a), foreign key fk(a) references t1(id));") + tk.MustExec("replace into t2 values (1, 1);") + tk.MustExec("replace into t2 (id) values (10);") + tk.MustGetDBError("replace into t2 values (3, 2);", plannercore.ErrNoReferencedRow2) +} + +func TestForeignKeyOnReplaceInto(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@foreign_key_checks=1") + tk.MustExec("use test") + tk.MustExec("create table t1 (id int key, a int, index (a));") + tk.MustExec("create table t2 (id int key, a int, index (a), constraint fk_1 foreign key (a) references t1(a));") + tk.MustExec("replace into t1 values (1, 1);") + tk.MustExec("replace into t2 values (1, 1);") + tk.MustExec("replace into t2 (id) values (2);") + tk.MustGetDBError("replace into t2 values (1, 2);", plannercore.ErrNoReferencedRow2) + // Test fk check on replace into parent table. + tk.MustGetDBError("replace into t1 values (1, 2);", plannercore.ErrRowIsReferenced2) + // Test fk cascade delete on replace into parent table. + tk.MustExec("alter table t2 drop foreign key fk_1") + tk.MustExec("alter table t2 add constraint fk_1 foreign key (a) references t1(a) on delete cascade") + tk.MustExec("replace into t1 values (1, 2);") + tk.MustQuery("select id, a from t1").Check(testkit.Rows("1 2")) + tk.MustQuery("select * from t2").Check(testkit.Rows("2 ")) + // Test fk cascade delete on replace into parent table. + tk.MustExec("alter table t2 drop foreign key fk_1") + tk.MustExec("alter table t2 add constraint fk_1 foreign key (a) references t1(a) on delete set null") + tk.MustExec("delete from t2") + tk.MustExec("delete from t1") + tk.MustExec("replace into t1 values (1, 1);") + tk.MustExec("replace into t2 values (1, 1);") + tk.MustExec("replace into t1 values (1, 2);") + tk.MustQuery("select id, a from t1").Check(testkit.Rows("1 2")) + tk.MustQuery("select id, a from t2").Check(testkit.Rows("1 ")) + + // Test cascade delete in self table by replace into statement. + tk.MustExec("drop table t1,t2") + tk.MustExec("create table t1 (id int key, name varchar(10), leader int, index(leader), foreign key (leader) references t1(id) ON DELETE CASCADE);") + tk.MustExec("replace into t1 values (1, 'boss', null), (10, 'l1_a', 1), (11, 'l1_b', 1), (12, 'l1_c', 1)") + tk.MustExec("replace into t1 values (100, 'l2_a1', 10), (101, 'l2_a2', 10), (102, 'l2_a3', 10)") + tk.MustExec("replace into t1 values (110, 'l2_b1', 11), (111, 'l2_b2', 11), (112, 'l2_b3', 11)") + tk.MustExec("replace into t1 values (120, 'l2_c1', 12), (121, 'l2_c2', 12), (122, 'l2_c3', 12)") + tk.MustExec("replace into t1 values (1000,'l3_a1', 100)") + tk.MustExec("replace into t1 values (1, 'new-boss', null)") + tk.MustQuery("select id from t1 order by id").Check(testkit.Rows("1")) +} diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go index 9a021568b20ee..600f052b1225e 100644 --- a/executor/index_lookup_join_test.go +++ b/executor/index_lookup_join_test.go @@ -428,17 +428,19 @@ PARTITIONS 1`) // Why does the t2.prefiller need be at least 2^32 ? If smaller the bug will not appear!?! tk.MustExec("insert into t2 values ( pow(2,32), 1, 1), ( pow(2,32)+1, 2, 0)") + tk.MustExec(`analyze table t1`) + tk.MustExec(`analyze table t2`) // Why must it be = 1 and not 2? - tk.MustQuery("explain select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows("" + - "IndexJoin_15 10.00 root inner join, inner:TableReader_14, outer key:test.t2.prefiller, inner key:test.t1.id, equal cond:eq(test.t2.prefiller, test.t1.id)]\n" + - "[├─HashAgg_25(Build) 8.00 root group by:test.t2.prefiller, funcs:firstrow(test.t2.prefiller)->test.t2.prefiller]\n" + - "[│ └─TableReader_26 8.00 root data:HashAgg_20]\n" + - "[│ └─HashAgg_20 8.00 cop[tikv] group by:test.t2.prefiller, ]\n" + - "[│ └─Selection_24 10.00 cop[tikv] eq(test.t2.postfiller, 1)]\n" + - "[│ └─TableFullScan_23 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo]\n" + - "[└─TableReader_14(Probe) 8.00 root partition:all data:TableRangeScan_13]\n" + - "[ └─TableRangeScan_13 8.00 cop[tikv] table:t1 range: decided by [eq(test.t1.id, test.t2.prefiller)], keep order:false, stats:pseudo")) + tk.MustQuery("explain format='brief' select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows(""+ + `IndexJoin 1.25 root inner join, inner:TableReader, outer key:test.t2.prefiller, inner key:test.t1.id, equal cond:eq(test.t2.prefiller, test.t1.id)`, + `├─HashAgg(Build) 1.00 root group by:test.t2.prefiller, funcs:firstrow(test.t2.prefiller)->test.t2.prefiller`, + `│ └─TableReader 1.00 root data:HashAgg`, + `│ └─HashAgg 1.00 cop[tikv] group by:test.t2.prefiller, `, + `│ └─Selection 1.00 cop[tikv] eq(test.t2.postfiller, 1)`, + `│ └─TableFullScan 2.00 cop[tikv] table:t2 keep order:false`, + `└─TableReader(Probe) 1.00 root partition:all data:TableRangeScan`, + ` └─TableRangeScan 1.00 cop[tikv] table:t1 range: decided by [eq(test.t1.id, test.t2.prefiller)], keep order:false, stats:pseudo`)) tk.MustQuery("show warnings").Check(testkit.Rows()) // without fix it fails with: "runtime error: index out of range [0] with length 0" tk.MustQuery("select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows()) diff --git a/executor/oomtest/oom_test.go b/executor/oomtest/oom_test.go index 5b348f5c238de..fc95bb47ceab8 100644 --- a/executor/oomtest/oom_test.go +++ b/executor/oomtest/oom_test.go @@ -223,7 +223,8 @@ func (h *oomCapture) Write(entry zapcore.Entry, fields []zapcore.Field) error { return nil } // They are just common background task and not related to the oom. - if entry.Message == "SetTiFlashGroupConfig" { + if entry.Message == "SetTiFlashGroupConfig" || + entry.Message == "record table item load status failed due to not finding item" { return nil } diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index 15d2c2872ca9c..5696b56f6f730 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -3831,3 +3831,72 @@ func TestIssue21732(t *testing.T) { }) } } + +func TestIssue39999(t *testing.T) { + store := testkit.CreateMockStore(t) + + tk := testkit.NewTestKit(t, store) + + tk.MustExec(`create schema test39999`) + tk.MustExec(`use test39999`) + tk.MustExec(`drop table if exists c, t`) + tk.MustExec("CREATE TABLE `c` (" + + "`serial_id` varchar(24)," + + "`occur_trade_date` date," + + "`txt_account_id` varchar(24)," + + "`capital_sub_class` varchar(10)," + + "`occur_amount` decimal(16,2)," + + "`broker` varchar(10)," + + "PRIMARY KEY (`txt_account_id`,`occur_trade_date`,`serial_id`) /*T![clustered_index] CLUSTERED */," + + "KEY `idx_serial_id` (`serial_id`)" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci " + + "PARTITION BY RANGE COLUMNS(`serial_id`) (" + + "PARTITION `p202209` VALUES LESS THAN ('20221001')," + + "PARTITION `p202210` VALUES LESS THAN ('20221101')," + + "PARTITION `p202211` VALUES LESS THAN ('20221201')" + + ")") + + tk.MustExec("CREATE TABLE `t` ( " + + "`txn_account_id` varchar(24), " + + "`account_id` varchar(32), " + + "`broker` varchar(10), " + + "PRIMARY KEY (`txn_account_id`) /*T![clustered_index] CLUSTERED */ " + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci") + + tk.MustExec("INSERT INTO `c` (serial_id, txt_account_id, capital_sub_class, occur_trade_date, occur_amount, broker) VALUES ('2022111700196920','04482786','CUST','2022-11-17',-2.01,'0009')") + tk.MustExec("INSERT INTO `t` VALUES ('04482786','1142927','0009')") + + tk.MustExec(`set tidb_partition_prune_mode='dynamic'`) + tk.MustExec(`analyze table c`) + tk.MustExec(`analyze table t`) + query := `select + /*+ inl_join(c) */ + c.occur_amount +from + c + join t on c.txt_account_id = t.txn_account_id + and t.broker = '0009' + and c.occur_trade_date = '2022-11-17'` + tk.MustQuery("explain " + query).Check(testkit.Rows(""+ + "IndexJoin_22 1.00 root inner join, inner:TableReader_21, outer key:test39999.t.txn_account_id, inner key:test39999.c.txt_account_id, equal cond:eq(test39999.t.txn_account_id, test39999.c.txt_account_id)", + "├─TableReader_27(Build) 1.00 root data:Selection_26", + "│ └─Selection_26 1.00 cop[tikv] eq(test39999.t.broker, \"0009\")", + "│ └─TableFullScan_25 1.00 cop[tikv] table:t keep order:false", + "└─TableReader_21(Probe) 1.00 root partition:all data:Selection_20", + " └─Selection_20 1.00 cop[tikv] eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)", + " └─TableRangeScan_19 1.00 cop[tikv] table:c range: decided by [eq(test39999.c.txt_account_id, test39999.t.txn_account_id) eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)], keep order:false")) + tk.MustQuery(query).Check(testkit.Rows("-2.01")) + + // Add the missing partition key part. + tk.MustExec(`alter table t add column serial_id varchar(24) default '2022111700196920'`) + query += ` and c.serial_id = t.serial_id` + tk.MustQuery(query).Check(testkit.Rows("-2.01")) + tk.MustQuery("explain " + query).Check(testkit.Rows(""+ + `IndexJoin_20 0.80 root inner join, inner:TableReader_19, outer key:test39999.t.txn_account_id, test39999.t.serial_id, inner key:test39999.c.txt_account_id, test39999.c.serial_id, equal cond:eq(test39999.t.serial_id, test39999.c.serial_id), eq(test39999.t.txn_account_id, test39999.c.txt_account_id)`, + `├─TableReader_25(Build) 0.80 root data:Selection_24`, + `│ └─Selection_24 0.80 cop[tikv] eq(test39999.t.broker, "0009"), not(isnull(test39999.t.serial_id))`, + `│ └─TableFullScan_23 1.00 cop[tikv] table:t keep order:false`, + `└─TableReader_19(Probe) 0.80 root partition:all data:Selection_18`, + ` └─Selection_18 0.80 cop[tikv] eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)`, + ` └─TableRangeScan_17 0.80 cop[tikv] table:c range: decided by [eq(test39999.c.txt_account_id, test39999.t.txn_account_id) eq(test39999.c.serial_id, test39999.t.serial_id) eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)], keep order:false`)) +} diff --git a/executor/plan_replayer.go b/executor/plan_replayer.go index 60a47991b48df..fae6273b3bd5e 100644 --- a/executor/plan_replayer.go +++ b/executor/plan_replayer.go @@ -125,7 +125,7 @@ func (e *PlanReplayerExec) registerCaptureTask(ctx context.Context) error { func (e *PlanReplayerExec) createFile() error { var err error - e.DumpInfo.File, e.DumpInfo.FileName, err = replayer.GeneratePlanReplayerFile() + e.DumpInfo.File, e.DumpInfo.FileName, err = replayer.GeneratePlanReplayerFile(false) if err != nil { return err } diff --git a/executor/prepared.go b/executor/prepared.go index a9dd9452e3c99..6a5025e0d539b 100644 --- a/executor/prepared.go +++ b/executor/prepared.go @@ -115,7 +115,7 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error { return err } } - stmt, p, paramCnt, err := plannercore.GeneratePlanCacheStmtWithAST(ctx, e.ctx, stmt0) + stmt, p, paramCnt, err := plannercore.GeneratePlanCacheStmtWithAST(ctx, e.ctx, stmt0.Text(), stmt0) if err != nil { return err } diff --git a/executor/replace.go b/executor/replace.go index 158a620fb300e..bfc70ebc4451c 100644 --- a/executor/replace.go +++ b/executor/replace.go @@ -92,6 +92,10 @@ func (e *ReplaceExec) removeRow(ctx context.Context, txn kv.Transaction, handle if err != nil { return false, err } + err = onRemoveRowForFK(e.ctx, oldRow, e.fkChecks, e.fkCascades) + if err != nil { + return false, err + } e.ctx.GetSessionVars().StmtCtx.AddAffectedRows(1) return false, nil } @@ -277,3 +281,18 @@ func (e *ReplaceExec) setMessage() { stmtCtx.SetMessage(msg) } } + +// GetFKChecks implements WithForeignKeyTrigger interface. +func (e *ReplaceExec) GetFKChecks() []*FKCheckExec { + return e.fkChecks +} + +// GetFKCascades implements WithForeignKeyTrigger interface. +func (e *ReplaceExec) GetFKCascades() []*FKCascadeExec { + return e.fkCascades +} + +// HasFKCascades implements WithForeignKeyTrigger interface. +func (e *ReplaceExec) HasFKCascades() bool { + return len(e.fkCascades) > 0 +} diff --git a/executor/seqtest/prepared_test.go b/executor/seqtest/prepared_test.go index 11dae5e570767..5edbed52b4e13 100644 --- a/executor/seqtest/prepared_test.go +++ b/executor/seqtest/prepared_test.go @@ -420,14 +420,14 @@ func TestPreparedInsert(t *testing.T) { err = counter.Write(pb) require.NoError(t, err) hit := pb.GetCounter().GetValue() - require.Equal(t, float64(1), hit) + require.Equal(t, float64(0), hit) // insert-values-stmt cannot use the plan cache } tk.MustExec(`set @a=3,@b=3; execute stmt_insert using @a, @b;`) if flag { err = counter.Write(pb) require.NoError(t, err) hit := pb.GetCounter().GetValue() - require.Equal(t, float64(2), hit) + require.Equal(t, float64(0), hit) } result := tk.MustQuery("select id, c1 from prepare_test where id = ?", 1) @@ -443,21 +443,21 @@ func TestPreparedInsert(t *testing.T) { err = counter.Write(pb) require.NoError(t, err) hit := pb.GetCounter().GetValue() - require.Equal(t, float64(2), hit) + require.Equal(t, float64(0), hit) } tk.MustExec(`set @a=2; execute stmt_insert_select using @a;`) if flag { err = counter.Write(pb) require.NoError(t, err) hit := pb.GetCounter().GetValue() - require.Equal(t, float64(3), hit) + require.Equal(t, float64(1), hit) } tk.MustExec(`set @a=3; execute stmt_insert_select using @a;`) if flag { err = counter.Write(pb) require.NoError(t, err) hit := pb.GetCounter().GetValue() - require.Equal(t, float64(4), hit) + require.Equal(t, float64(2), hit) } result = tk.MustQuery("select id, c1 from prepare_test where id = ?", 101) diff --git a/executor/simple_test.go b/executor/simple_test.go index 61688519b55d5..a67933bbbca92 100644 --- a/executor/simple_test.go +++ b/executor/simple_test.go @@ -21,13 +21,13 @@ import ( "testing" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/auth" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/server" "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/util" "github.com/stretchr/testify/require" - tikvutil "github.com/tikv/client-go/v2/util" ) func TestKillStmt(t *testing.T) { @@ -86,7 +86,7 @@ func TestKillStmt(t *testing.T) { func TestUserAttributes(t *testing.T) { store, _ := testkit.CreateMockStoreAndDomain(t) rootTK := testkit.NewTestKit(t, store) - ctx := context.WithValue(context.Background(), tikvutil.RequestSourceKey, tikvutil.RequestSource{RequestSourceInternal: true}) + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnPrivilege) // https://dev.mysql.com/doc/refman/8.0/en/create-user.html#create-user-comments-attributes rootTK.MustExec(`CREATE USER testuser COMMENT '1234'`) diff --git a/expression/BUILD.bazel b/expression/BUILD.bazel index c7304642c544a..5a201d906b5a3 100644 --- a/expression/BUILD.bazel +++ b/expression/BUILD.bazel @@ -177,6 +177,7 @@ go_test( "integration_serial_test.go", "integration_test.go", "main_test.go", + "multi_valued_index_test.go", "scalar_function_test.go", "schema_test.go", "typeinfer_test.go", diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index ee66669e638d6..e6257c4dd058c 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -23,6 +23,7 @@ package expression import ( + "fmt" "math" "strconv" "strings" @@ -407,6 +408,70 @@ func (c *castAsDurationFunctionClass) getFunction(ctx sessionctx.Context, args [ return sig, nil } +type castAsArrayFunctionClass struct { + baseFunctionClass + + tp *types.FieldType +} + +func (c *castAsArrayFunctionClass) verifyArgs(args []Expression) error { + if err := c.baseFunctionClass.verifyArgs(args); err != nil { + return err + } + + if args[0].GetType().EvalType() != types.ETJson { + return types.ErrInvalidJSONData.GenWithStackByArgs("1", "cast_as_array") + } + + return nil +} + +func (c *castAsArrayFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (sig builtinFunc, err error) { + if err := c.verifyArgs(args); err != nil { + return nil, err + } + arrayType := c.tp.ArrayType() + switch arrayType.GetType() { + case mysql.TypeYear, mysql.TypeJSON: + return nil, ErrNotSupportedYet.GenWithStackByArgs(fmt.Sprintf("CAST-ing data to array of %s", arrayType.String())) + } + if arrayType.EvalType() == types.ETString && arrayType.GetCharset() != charset.CharsetUTF8MB4 && arrayType.GetCharset() != charset.CharsetBin { + return nil, ErrNotSupportedYet.GenWithStackByArgs("specifying charset for multi-valued index", arrayType.String()) + } + + bf, err := newBaseBuiltinFunc(ctx, c.funcName, args, c.tp) + if err != nil { + return nil, err + } + sig = &castJSONAsArrayFunctionSig{bf} + return sig, nil +} + +type castJSONAsArrayFunctionSig struct { + baseBuiltinFunc +} + +func (b *castJSONAsArrayFunctionSig) Clone() builtinFunc { + newSig := &castJSONAsArrayFunctionSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +func (b *castJSONAsArrayFunctionSig) evalJSON(row chunk.Row) (res types.BinaryJSON, isNull bool, err error) { + val, isNull, err := b.args[0].EvalJSON(b.ctx, row) + if isNull || err != nil { + return res, isNull, err + } + + if val.TypeCode != types.JSONTypeCodeArray { + return types.BinaryJSON{}, false, ErrNotSupportedYet.GenWithStackByArgs("CAST-ing Non-JSON Array type to array") + } + + // TODO: impl the cast(... as ... array) function + + return types.BinaryJSON{}, false, nil +} + type castAsJSONFunctionClass struct { baseFunctionClass @@ -1914,6 +1979,13 @@ func BuildCastCollationFunction(ctx sessionctx.Context, expr Expression, ec *Exp // BuildCastFunction builds a CAST ScalarFunction from the Expression. func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldType) (res Expression) { + res, err := BuildCastFunctionWithCheck(ctx, expr, tp) + terror.Log(err) + return +} + +// BuildCastFunctionWithCheck builds a CAST ScalarFunction from the Expression and return error if any. +func BuildCastFunctionWithCheck(ctx sessionctx.Context, expr Expression, tp *types.FieldType) (res Expression, err error) { argType := expr.GetType() // If source argument's nullable, then target type should be nullable if !mysql.HasNotNullFlag(argType.GetFlag()) { @@ -1933,7 +2005,11 @@ func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldT case types.ETDuration: fc = &castAsDurationFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp} case types.ETJson: - fc = &castAsJSONFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp} + if tp.IsArray() { + fc = &castAsArrayFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp} + } else { + fc = &castAsJSONFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp} + } case types.ETString: fc = &castAsStringFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp} if expr.GetType().GetType() == mysql.TypeBit { @@ -1941,7 +2017,6 @@ func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldT } } f, err := fc.getFunction(ctx, []Expression{expr}) - terror.Log(err) res = &ScalarFunction{ FuncName: model.NewCIStr(ast.Cast), RetType: tp, @@ -1950,10 +2025,10 @@ func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldT // We do not fold CAST if the eval type of this scalar function is ETJson // since we may reset the flag of the field type of CastAsJson later which // would affect the evaluation of it. - if tp.EvalType() != types.ETJson { + if tp.EvalType() != types.ETJson && err == nil { res = FoldConstant(res) } - return res + return res, err } // WrapWithCastAsInt wraps `expr` with `cast` if the return type of expr is not diff --git a/expression/errors.go b/expression/errors.go index 0db38645f78d4..c56737ec2fae3 100644 --- a/expression/errors.go +++ b/expression/errors.go @@ -37,6 +37,7 @@ var ( ErrInvalidTableSample = dbterror.ClassExpression.NewStd(mysql.ErrInvalidTableSample) ErrInternal = dbterror.ClassOptimizer.NewStd(mysql.ErrInternal) ErrNoDB = dbterror.ClassOptimizer.NewStd(mysql.ErrNoDB) + ErrNotSupportedYet = dbterror.ClassExpression.NewStd(mysql.ErrNotSupportedYet) // All the un-exported errors are defined here: errFunctionNotExists = dbterror.ClassExpression.NewStd(mysql.ErrSpDoesNotExist) diff --git a/expression/expression.go b/expression/expression.go index 024bac00ef960..ee4ba67f1f854 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -59,7 +59,7 @@ var EvalAstExpr func(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, e // RewriteAstExpr rewrites ast expression directly. // Note: initialized in planner/core // import expression and planner/core together to use EvalAstExpr -var RewriteAstExpr func(sctx sessionctx.Context, expr ast.ExprNode, schema *Schema, names types.NameSlice) (Expression, error) +var RewriteAstExpr func(sctx sessionctx.Context, expr ast.ExprNode, schema *Schema, names types.NameSlice, allowCastArray bool) (Expression, error) // VecExpr contains all vectorized evaluation methods. type VecExpr interface { @@ -998,7 +998,7 @@ func ColumnInfos2ColumnsAndNames(ctx sessionctx.Context, dbName, tblName model.C if err != nil { return nil, nil, errors.Trace(err) } - e, err := RewriteAstExpr(ctx, expr, mockSchema, names) + e, err := RewriteAstExpr(ctx, expr, mockSchema, names, true) if err != nil { return nil, nil, errors.Trace(err) } @@ -1358,12 +1358,15 @@ func canScalarFuncPushDown(scalarFunc *ScalarFunction, pc PbConverter, storeType panic(errors.Errorf("unspecified PbCode: %T", scalarFunc.Function)) }) } + storageName := storeType.Name() + if storeType == kv.UnSpecified { + storageName = "storage layer" + } + warnErr := errors.New("Scalar function '" + scalarFunc.FuncName.L + "'(signature: " + scalarFunc.Function.PbCode().String() + ", return type: " + scalarFunc.RetType.CompactStr() + ") is not supported to push down to " + storageName + " now.") if pc.sc.InExplainStmt { - storageName := storeType.Name() - if storeType == kv.UnSpecified { - storageName = "storage layer" - } - pc.sc.AppendWarning(errors.New("Scalar function '" + scalarFunc.FuncName.L + "'(signature: " + scalarFunc.Function.PbCode().String() + ", return type: " + scalarFunc.RetType.CompactStr() + ") is not supported to push down to " + storageName + " now.")) + pc.sc.AppendWarning(warnErr) + } else { + pc.sc.AppendExtraWarning(warnErr) } return false } @@ -1393,14 +1396,20 @@ func canExprPushDown(expr Expression, pc PbConverter, storeType kv.StoreType, ca if expr.GetType().GetType() == mysql.TypeEnum && canEnumPush { break } + warnErr := errors.New("Expression about '" + expr.String() + "' can not be pushed to TiFlash because it contains unsupported calculation of type '" + types.TypeStr(expr.GetType().GetType()) + "'.") if pc.sc.InExplainStmt { - pc.sc.AppendWarning(errors.New("Expression about '" + expr.String() + "' can not be pushed to TiFlash because it contains unsupported calculation of type '" + types.TypeStr(expr.GetType().GetType()) + "'.")) + pc.sc.AppendWarning(warnErr) + } else { + pc.sc.AppendExtraWarning(warnErr) } return false case mysql.TypeNewDecimal: if !expr.GetType().IsDecimalValid() { + warnErr := errors.New("Expression about '" + expr.String() + "' can not be pushed to TiFlash because it contains invalid decimal('" + strconv.Itoa(expr.GetType().GetFlen()) + "','" + strconv.Itoa(expr.GetType().GetDecimal()) + "').") if pc.sc.InExplainStmt { - pc.sc.AppendWarning(errors.New("Expression about '" + expr.String() + "' can not be pushed to TiFlash because it contains invalid decimal('" + strconv.Itoa(expr.GetType().GetFlen()) + "','" + strconv.Itoa(expr.GetType().GetDecimal()) + "').")) + pc.sc.AppendWarning(warnErr) + } else { + pc.sc.AppendExtraWarning(warnErr) } return false } diff --git a/expression/multi_valued_index_test.go b/expression/multi_valued_index_test.go new file mode 100644 index 0000000000000..788dadddab02c --- /dev/null +++ b/expression/multi_valued_index_test.go @@ -0,0 +1,58 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expression_test + +import ( + "testing" + + "github.com/pingcap/tidb/errno" + "github.com/pingcap/tidb/testkit" +) + +func TestMultiValuedIndexDDL(t *testing.T) { + store := testkit.CreateMockStore(t) + + tk := testkit.NewTestKit(t, store) + tk.MustExec("USE test;") + + tk.MustExec("create table t(a json);") + tk.MustGetErrCode("select cast(a as signed array) from t", errno.ErrNotSupportedYet) + tk.MustGetErrCode("select json_extract(cast(a as signed array), '$[0]') from t", errno.ErrNotSupportedYet) + tk.MustGetErrCode("select * from t where cast(a as signed array)", errno.ErrNotSupportedYet) + tk.MustGetErrCode("select cast('[1,2,3]' as unsigned array);", errno.ErrNotSupportedYet) + + tk.MustExec("drop table t") + tk.MustGetErrCode("CREATE TABLE t(x INT, KEY k ((1 AND CAST(JSON_ARRAY(x) AS UNSIGNED ARRAY))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(cast(f1 as unsigned array) as unsigned array))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->>'$[*]' as unsigned array))));", errno.ErrInvalidJSONData) + tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->'$[*]' as year array))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->'$[*]' as json array))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->'$[*]' as char(10) charset gbk array))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("create table t(j json, gc json as ((concat(cast(j->'$[*]' as unsigned array),\"x\"))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("create table t(j json, gc json as (cast(j->'$[*]' as unsigned array)));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("create view v as select cast('[1,2,3]' as unsigned array);", errno.ErrNotSupportedYet) + tk.MustExec("create table t(a json, index idx((cast(a as signed array))));") + + tk.MustExec("drop table t") + tk.MustGetErrCode("create table t(a json, b int, index idx(b, (cast(a as signed array)), (cast(a as signed array))));", errno.ErrNotSupportedYet) + tk.MustExec("create table t(a json, b int);") + tk.MustGetErrCode("create index idx on t (b, (cast(a as signed array)), (cast(a as signed array)))", errno.ErrNotSupportedYet) + tk.MustGetErrCode("alter table t add index idx(b, (cast(a as signed array)), (cast(a as signed array)))", errno.ErrNotSupportedYet) + tk.MustExec("create index idx1 on t (b, (cast(a as signed array)))") + tk.MustExec("alter table t add index idx2(b, (cast(a as signed array)))") + + tk.MustExec("drop table t") + tk.MustExec("create table t(a json, b int, index idx3(b, (cast(a as signed array))));") +} diff --git a/expression/simple_rewriter.go b/expression/simple_rewriter.go index 808db9f69b4cf..3343a0cbaa169 100644 --- a/expression/simple_rewriter.go +++ b/expression/simple_rewriter.go @@ -48,7 +48,7 @@ func ParseSimpleExprWithTableInfo(ctx sessionctx.Context, exprStr string, tableI return nil, errors.Trace(err) } expr := stmts[0].(*ast.SelectStmt).Fields.Fields[0].Expr - return RewriteSimpleExprWithTableInfo(ctx, tableInfo, expr) + return RewriteSimpleExprWithTableInfo(ctx, tableInfo, expr, false) } // ParseSimpleExprCastWithTableInfo parses simple expression string to Expression. @@ -63,13 +63,13 @@ func ParseSimpleExprCastWithTableInfo(ctx sessionctx.Context, exprStr string, ta } // RewriteSimpleExprWithTableInfo rewrites simple ast.ExprNode to expression.Expression. -func RewriteSimpleExprWithTableInfo(ctx sessionctx.Context, tbl *model.TableInfo, expr ast.ExprNode) (Expression, error) { +func RewriteSimpleExprWithTableInfo(ctx sessionctx.Context, tbl *model.TableInfo, expr ast.ExprNode, allowCastArray bool) (Expression, error) { dbName := model.NewCIStr(ctx.GetSessionVars().CurrentDB) columns, names, err := ColumnInfos2ColumnsAndNames(ctx, dbName, tbl.Name, tbl.Cols(), tbl) if err != nil { return nil, err } - e, err := RewriteAstExpr(ctx, expr, NewSchema(columns...), names) + e, err := RewriteAstExpr(ctx, expr, NewSchema(columns...), names, allowCastArray) if err != nil { return nil, err } @@ -111,7 +111,7 @@ func ParseSimpleExprsWithNames(ctx sessionctx.Context, exprStr string, schema *S // RewriteSimpleExprWithNames rewrites simple ast.ExprNode to expression.Expression. func RewriteSimpleExprWithNames(ctx sessionctx.Context, expr ast.ExprNode, schema *Schema, names []*types.FieldName) (Expression, error) { - e, err := RewriteAstExpr(ctx, expr, schema, names) + e, err := RewriteAstExpr(ctx, expr, schema, names, false) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 223d775072539..0fe0c295d9cb3 100644 --- a/go.mod +++ b/go.mod @@ -116,7 +116,7 @@ require ( golang.org/x/sys v0.3.0 golang.org/x/term v0.3.0 golang.org/x/text v0.5.0 - golang.org/x/time v0.2.0 + golang.org/x/time v0.3.0 golang.org/x/tools v0.2.0 google.golang.org/api v0.74.0 google.golang.org/grpc v1.45.0 diff --git a/go.sum b/go.sum index 7b939d5d1109e..6d5af4dc711ef 100644 --- a/go.sum +++ b/go.sum @@ -1340,8 +1340,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= -golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/infoschema/cluster_tables_test.go b/infoschema/cluster_tables_test.go index 00892c95cfe9c..73488028a26ec 100644 --- a/infoschema/cluster_tables_test.go +++ b/infoschema/cluster_tables_test.go @@ -1028,3 +1028,34 @@ func TestBindingFromHistoryWithTiFlashBindable(t *testing.T) { planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows() tk.MustGetErrMsg(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0]), "can't create binding for query with tiflash engine") } + +func TestSetBindingStatusBySQLDigest(t *testing.T) { + s := new(clusterTablesSuite) + s.store, s.dom = testkit.CreateMockStoreAndDomain(t) + s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil) + s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer() + s.startTime = time.Now() + defer s.httpServer.Close() + defer s.rpcserver.Stop() + tk := s.newTestKitWithRoot(t) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil)) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(id int, a int, key(a))") + sql := "select /*+ ignore_index(t, a) */ * from t where t.a = 1" + tk.MustExec(sql) + planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.cluster_statements_summary where query_sample_text = '%s'", sql)).Rows() + tk.MustExec(fmt.Sprintf("create global binding from history using plan digest '%s'", planDigest[0][0])) + sql = "select * from t where t.a = 1" + tk.MustExec(sql) + tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1")) + + sqlDigest := tk.MustQuery("show global bindings").Rows() + tk.MustExec(fmt.Sprintf("set binding disabled for sql digest '%s'", sqlDigest[0][9])) + tk.MustExec(sql) + tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("0")) + tk.MustExec(fmt.Sprintf("set binding enabled for sql digest '%s'", sqlDigest[0][9])) + tk.MustExec(sql) + tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1")) + tk.MustGetErrMsg("set binding enabled for sql digest '2'", "can't find any binding for '2'") +} diff --git a/infoschema/error.go b/infoschema/error.go index a7e4929a35bcc..d89d3e3dc8201 100644 --- a/infoschema/error.go +++ b/infoschema/error.go @@ -64,6 +64,8 @@ var ( ErrKeyNotExists = dbterror.ClassSchema.NewStd(mysql.ErrKeyDoesNotExist) // ErrCannotAddForeign returns for foreign key exists. ErrCannotAddForeign = dbterror.ClassSchema.NewStd(mysql.ErrCannotAddForeign) + // ErrForeignKeyOnPartitioned returns for foreign key on partition table. + ErrForeignKeyOnPartitioned = dbterror.ClassSchema.NewStd(mysql.ErrForeignKeyOnPartitioned) // ErrForeignKeyNotMatch returns for foreign key not match. ErrForeignKeyNotMatch = dbterror.ClassSchema.NewStd(mysql.ErrWrongFkDef) // ErrIndexExists returns for index already exists. diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index ebf3e9b535893..d4d4c4fa588f7 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -531,18 +531,166 @@ func TestSlowQuery(t *testing.T) { slowLogFileName := "tidb_slow.log" prepareSlowLogfile(t, slowLogFileName) defer func() { require.NoError(t, os.Remove(slowLogFileName)) }() + expectedRes := [][]interface{}{ + {"2019-02-12 19:33:56.571953", + "406315658548871171", + "root", + "localhost", + "6", + "57", + "0.12", + "4.895492", + "0.4", + "0.2", + "0.000000003", + "2", + "0.000000002", + "0.00000001", + "0.000000003", + "0.19", + "0.21", + "0.01", + "0", + "0.18", + "[txnLock]", + "0.03", + "0", + "15", + "480", + "1", + "8", + "0.3824278", + "0.161", + "0.101", + "0.092", + "1.71", + "1", + "100001", + "100000", + "100", + "10", + "10", + "10", + "100", + "test", + "", + "0", + "42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772", + "t1:1,t2:2", + "0.1", + "0.2", + "0.03", + "127.0.0.1:20160", + "0.05", + "0.6", + "0.8", + "0.0.0.0:20160", + "70724", + "65536", + "0", + "0", + "0", + "0", + "10", + "", + "0", + "1", + "0", + "0", + "1", + "0", + "0", + "abcd", + "60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4", + "", + "update t set i = 2;", + "select * from t_slim;"}, + {"2021-09-08 14:39:54.506967", + "427578666238083075", + "root", + "172.16.0.0", + "40507", + "0", + "0", + "25.571605962", + "0.002923536", + "0.006800973", + "0.002100764", + "0", + "0", + "0", + "0.000015801", + "25.542014572", + "0", + "0.002294647", + "0.000605473", + "12.483", + "[tikvRPC regionMiss tikvRPC regionMiss regionMiss]", + "0", + "0", + "624", + "172064", + "60", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "rtdb", + "", + "0", + "124acb3a0bec903176baca5f9da00b4e7512a41c93b417923f26502edeb324cc", + "", + "0", + "0", + "0", + "", + "0", + "0", + "0", + "", + "856544", + "0", + "86.635049185", + "0.015486658", + "100.054", + "0", + "0", + "", + "0", + "1", + "0", + "0", + "0", + "0", + "0", + "", + "", + "", + "", + "INSERT INTO ...;", + }, + } tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", slowLogFileName)) tk.MustExec("set time_zone = '+08:00';") re := tk.MustQuery("select * from information_schema.slow_query") - re.Check(testkit.RowsWithSep("|", "2019-02-12 19:33:56.571953|406315658548871171|root|localhost|6|57|0.12|4.895492|0.4|0.2|0.000000003|2|0.000000002|0.00000001|0.000000003|0.19|0.21|0.01|0|0.18|[txnLock]|0.03|0|15|480|1|8|0.3824278|0.161|0.101|0.092|1.71|1|100001|100000|100|10|10|10|100|test||0|42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772|t1:1,t2:2|0.1|0.2|0.03|127.0.0.1:20160|0.05|0.6|0.8|0.0.0.0:20160|70724|65536|0|0|0|0|10||0|1|0|0|1|0|0|abcd|60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4||update t set i = 2;|select * from t_slim;", - "2021-09-08|14:39:54.506967|427578666238083075|root|172.16.0.0|40507|0|0|25.571605962|0.002923536|0.006800973|0.002100764|0|0|0|0.000015801|25.542014572|0|0.002294647|0.000605473|12.483|[tikvRPC regionMiss tikvRPC regionMiss regionMiss]|0|0|624|172064|60|0|0|0|0|0|0|0|0|0|0|0|0|0|0|rtdb||0|124acb3a0bec903176baca5f9da00b4e7512a41c93b417923f26502edeb324cc||0|0|0||0|0|0||856544|0|86.635049185|0.015486658|100.054|0|0||0|1|0|0|0|0|0|||||INSERT INTO ...;", - )) + re.Check(expectedRes) + tk.MustExec("set time_zone = '+00:00';") re = tk.MustQuery("select * from information_schema.slow_query") - re.Check(testkit.RowsWithSep("|", "2019-02-12 11:33:56.571953|406315658548871171|root|localhost|6|57|0.12|4.895492|0.4|0.2|0.000000003|2|0.000000002|0.00000001|0.000000003|0.19|0.21|0.01|0|0.18|[txnLock]|0.03|0|15|480|1|8|0.3824278|0.161|0.101|0.092|1.71|1|100001|100000|100|10|10|10|100|test||0|42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772|t1:1,t2:2|0.1|0.2|0.03|127.0.0.1:20160|0.05|0.6|0.8|0.0.0.0:20160|70724|65536|0|0|0|0|10||0|1|0|0|1|0|0|abcd|60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4||update t set i = 2;|select * from t_slim;", - "2021-09-08|06:39:54.506967|427578666238083075|root|172.16.0.0|40507|0|0|25.571605962|0.002923536|0.006800973|0.002100764|0|0|0|0.000015801|25.542014572|0|0.002294647|0.000605473|12.483|[tikvRPC regionMiss tikvRPC regionMiss regionMiss]|0|0|624|172064|60|0|0|0|0|0|0|0|0|0|0|0|0|0|0|rtdb||0|124acb3a0bec903176baca5f9da00b4e7512a41c93b417923f26502edeb324cc||0|0|0||0|0|0||856544|0|86.635049185|0.015486658|100.054|0|0||0|1|0|0|0|0|0|||||INSERT INTO ...;", - )) + expectedRes[0][0] = "2019-02-12 11:33:56.571953" + expectedRes[1][0] = "2021-09-08 06:39:54.506967" + re.Check(expectedRes) // Test for long query. f, err := os.OpenFile(slowLogFileName, os.O_CREATE|os.O_WRONLY, 0644) diff --git a/kv/mpp.go b/kv/mpp.go index b0752f8186deb..2e398af595650 100644 --- a/kv/mpp.go +++ b/kv/mpp.go @@ -16,7 +16,6 @@ package kv import ( "context" - "sync" "time" "github.com/pingcap/kvproto/pkg/mpp" @@ -81,7 +80,7 @@ type MPPDispatchRequest struct { type MPPClient interface { // ConstructMPPTasks schedules task for a plan fragment. // TODO:: This interface will be refined after we support more executors. - ConstructMPPTasks(context.Context, *MPPBuildTasksRequest, *sync.Map, time.Duration) ([]MPPTaskMeta, error) + ConstructMPPTasks(context.Context, *MPPBuildTasksRequest, time.Duration) ([]MPPTaskMeta, error) // DispatchMPPTasks dispatches ALL mpp requests at once, and returns an iterator that transfers the data. DispatchMPPTasks(ctx context.Context, vars interface{}, reqs []*MPPDispatchRequest, needTriggerFallback bool, startTs uint64) Response diff --git a/kv/txn.go b/kv/txn.go index d7828c7fb3138..035f2aa662eca 100644 --- a/kv/txn.go +++ b/kv/txn.go @@ -195,20 +195,22 @@ func BackOff(attempts uint) int { func setRequestSourceForInnerTxn(ctx context.Context, txn Transaction) { if source := ctx.Value(RequestSourceKey); source != nil { requestSource := source.(RequestSource) - if !requestSource.RequestSourceInternal { - logutil.Logger(ctx).Warn("`RunInNewTxn` should be used by inner txn only") + if requestSource.RequestSourceType != "" { + if !requestSource.RequestSourceInternal { + logutil.Logger(ctx).Warn("`RunInNewTxn` should be used by inner txn only") + } + txn.SetOption(RequestSourceInternal, requestSource.RequestSourceInternal) + txn.SetOption(RequestSourceType, requestSource.RequestSourceType) + return } - txn.SetOption(RequestSourceInternal, requestSource.RequestSourceInternal) - txn.SetOption(RequestSourceType, requestSource.RequestSourceType) + } + // panic in test mode in case there are requests without source in the future. + // log warnings in production mode. + if flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil { + panic("unexpected no source type context, if you see this error, " + + "the `RequestSourceTypeKey` is missing in your context") } else { - // panic in test mode in case there are requests without source in the future. - // log warnings in production mode. - if flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil { - panic("unexpected no source type context, if you see this error, " + - "the `RequestSourceTypeKey` is missing in your context") - } else { - logutil.Logger(ctx).Warn("unexpected no source type context, if you see this warning, " + - "the `RequestSourceTypeKey` is missing in the context") - } + logutil.Logger(ctx).Warn("unexpected no source type context, if you see this warning, " + + "the `RequestSourceTypeKey` is missing in the context") } } diff --git a/metrics/grafana/tidb.json b/metrics/grafana/tidb.json index 7677a718e156f..0e2ce93934f7c 100644 --- a/metrics/grafana/tidb.json +++ b/metrics/grafana/tidb.json @@ -17000,6 +17000,212 @@ "title": "TTL", "type": "row" }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 291, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 289, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "tidb_server_gogc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GOGC", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "exponential moving average of CPU Usage", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 293, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "tidb_rm_ema_cpu_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "EMA CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Resource Manager", + "type": "row" + }, { "aliasColors": {}, "bars": false, diff --git a/metrics/metrics.go b/metrics/metrics.go index 0cca442e5eb83..f609caaa124cd 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -182,6 +182,7 @@ func RegisterMetrics() { prometheus.MustRegister(TokenGauge) prometheus.MustRegister(ConfigStatus) prometheus.MustRegister(TiFlashQueryTotalCounter) + prometheus.MustRegister(TiFlashFailedMPPStoreState) prometheus.MustRegister(SmallTxnWriteDuration) prometheus.MustRegister(TxnWriteThroughput) prometheus.MustRegister(LoadSysVarCacheCounter) @@ -238,6 +239,7 @@ func ToggleSimplifiedMode(simplified bool) { InfoCacheCounters, ReadFromTableCacheCounter, TiFlashQueryTotalCounter, + TiFlashFailedMPPStoreState, CampaignOwnerCounter, NonTransactionalDMLCount, MemoryUsage, diff --git a/metrics/server.go b/metrics/server.go index 116b02eb122b6..6bd34c4a8b903 100644 --- a/metrics/server.go +++ b/metrics/server.go @@ -279,6 +279,14 @@ var ( Help: "Counter of TiFlash queries.", }, []string{LblType, LblResult}) + TiFlashFailedMPPStoreState = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "server", + Name: "tiflash_failed_store", + Help: "Statues of failed tiflash mpp store,-1 means detector heartbeat,0 means reachable,1 means abnormal.", + }, []string{LblAddress}) + PDAPIExecutionHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "tidb", diff --git a/parser/ast/misc.go b/parser/ast/misc.go index 048b0d9c65b52..7a0e2fc7a1a50 100644 --- a/parser/ast/misc.go +++ b/parser/ast/misc.go @@ -3768,7 +3768,7 @@ func (n *TableOptimizerHint) Restore(ctx *format.RestoreCtx) error { } table.Restore(ctx) } - case "use_index", "ignore_index", "use_index_merge", "force_index": + case "use_index", "ignore_index", "use_index_merge", "force_index", "keep_order", "no_keep_order": n.Tables[0].Restore(ctx) ctx.WritePlain(" ") for i, index := range n.Indexes { diff --git a/parser/ast/misc_test.go b/parser/ast/misc_test.go index 8042a171874d0..36e8cb1ef002f 100644 --- a/parser/ast/misc_test.go +++ b/parser/ast/misc_test.go @@ -228,6 +228,18 @@ func TestTableOptimizerHintRestore(t *testing.T) { {"IGNORE_INDEX(@sel_1 t1 c1)", "IGNORE_INDEX(@`sel_1` `t1` `c1`)"}, {"IGNORE_INDEX(t1@sel_1 c1)", "IGNORE_INDEX(`t1`@`sel_1` `c1`)"}, {"IGNORE_INDEX(t1@sel_1 partition(p0, p1) c1)", "IGNORE_INDEX(`t1`@`sel_1` PARTITION(`p0`, `p1`) `c1`)"}, + {"KEEP_ORDER(t1 c1)", "KEEP_ORDER(`t1` `c1`)"}, + {"KEEP_ORDER(test.t1 c1)", "KEEP_ORDER(`test`.`t1` `c1`)"}, + {"KEEP_ORDER(@sel_1 t1 c1)", "KEEP_ORDER(@`sel_1` `t1` `c1`)"}, + {"KEEP_ORDER(t1@sel_1 c1)", "KEEP_ORDER(`t1`@`sel_1` `c1`)"}, + {"KEEP_ORDER(test.t1@sel_1 c1)", "KEEP_ORDER(`test`.`t1`@`sel_1` `c1`)"}, + {"KEEP_ORDER(test.t1@sel_1 partition(p0) c1)", "KEEP_ORDER(`test`.`t1`@`sel_1` PARTITION(`p0`) `c1`)"}, + {"NO_KEEP_ORDER(t1 c1)", "NO_KEEP_ORDER(`t1` `c1`)"}, + {"NO_KEEP_ORDER(test.t1 c1)", "NO_KEEP_ORDER(`test`.`t1` `c1`)"}, + {"NO_KEEP_ORDER(@sel_1 t1 c1)", "NO_KEEP_ORDER(@`sel_1` `t1` `c1`)"}, + {"NO_KEEP_ORDER(t1@sel_1 c1)", "NO_KEEP_ORDER(`t1`@`sel_1` `c1`)"}, + {"NO_KEEP_ORDER(test.t1@sel_1 c1)", "NO_KEEP_ORDER(`test`.`t1`@`sel_1` `c1`)"}, + {"NO_KEEP_ORDER(test.t1@sel_1 partition(p0) c1)", "NO_KEEP_ORDER(`test`.`t1`@`sel_1` PARTITION(`p0`) `c1`)"}, {"TIDB_SMJ(`t1`)", "TIDB_SMJ(`t1`)"}, {"TIDB_SMJ(t1)", "TIDB_SMJ(`t1`)"}, {"TIDB_SMJ(t1,t2)", "TIDB_SMJ(`t1`, `t2`)"}, diff --git a/parser/hintparser.go b/parser/hintparser.go index 1b14e7d292e1e..a619d2a1c74f2 100644 --- a/parser/hintparser.go +++ b/parser/hintparser.go @@ -41,18 +41,18 @@ type yyhintXError struct { } const ( - yyhintDefault = 57424 + yyhintDefault = 57426 yyhintEOFCode = 57344 yyhintErrCode = 57345 hintAggToCop = 57379 hintBCJoin = 57394 hintBKA = 57355 hintBNL = 57357 - hintDupsWeedOut = 57420 - hintFalse = 57416 - hintFirstMatch = 57421 - hintForceIndex = 57406 - hintGB = 57419 + hintDupsWeedOut = 57422 + hintFalse = 57418 + hintFirstMatch = 57423 + hintForceIndex = 57408 + hintGB = 57421 hintHashAgg = 57381 hintHashJoin = 57359 hintHashJoinBuild = 57360 @@ -70,12 +70,13 @@ const ( hintJoinOrder = 57352 hintJoinPrefix = 57353 hintJoinSuffix = 57354 - hintLeading = 57408 - hintLimitToCop = 57405 - hintLooseScan = 57422 - hintMB = 57418 + hintKeepOrder = 57400 + hintLeading = 57410 + hintLimitToCop = 57407 + hintLooseScan = 57424 + hintMB = 57420 hintMRR = 57367 - hintMaterialization = 57423 + hintMaterialization = 57425 hintMaxExecutionTime = 57375 hintMemoryQuota = 57388 hintMerge = 57363 @@ -83,175 +84,178 @@ const ( hintMpp2PhaseAgg = 57383 hintNoBKA = 57356 hintNoBNL = 57358 - hintNoDecorrelate = 57410 + hintNoDecorrelate = 57412 hintNoHashJoin = 57362 hintNoICP = 57369 hintNoIndexMerge = 57366 + hintNoKeepOrder = 57401 hintNoMRR = 57368 hintNoMerge = 57364 hintNoRangeOptimization = 57370 hintNoSemijoin = 57374 hintNoSkipScan = 57372 hintNoSwapJoinInputs = 57389 - hintNthPlan = 57404 - hintOLAP = 57411 - hintOLTP = 57412 - hintPartition = 57413 + hintNthPlan = 57406 + hintOLAP = 57413 + hintOLTP = 57414 + hintPartition = 57415 hintQBName = 57378 hintQueryType = 57390 hintReadConsistentReplica = 57391 hintReadFromStorage = 57392 hintResourceGroup = 57377 hintSMJoin = 57393 - hintSemiJoinRewrite = 57409 + hintSemiJoinRewrite = 57411 hintSemijoin = 57373 hintSetVar = 57376 hintShuffleJoin = 57395 hintSingleAtIdentifier = 57349 hintSkipScan = 57371 - hintStraightJoin = 57407 + hintStraightJoin = 57409 hintStreamAgg = 57396 hintStringLit = 57350 hintSwapJoinInputs = 57397 - hintTiFlash = 57415 - hintTiKV = 57414 - hintTimeRange = 57402 - hintTrue = 57417 - hintUseCascades = 57403 + hintTiFlash = 57417 + hintTiKV = 57416 + hintTimeRange = 57404 + hintTrue = 57419 + hintUseCascades = 57405 hintUseIndex = 57399 hintUseIndexMerge = 57398 - hintUsePlanCache = 57400 - hintUseToja = 57401 + hintUsePlanCache = 57402 + hintUseToja = 57403 yyhintMaxDepth = 200 - yyhintTabOfs = -193 + yyhintTabOfs = -197 ) var ( yyhintXLAT = map[int]int{ - 41: 0, // ')' (146x) - 57379: 1, // hintAggToCop (133x) - 57394: 2, // hintBCJoin (133x) - 57355: 3, // hintBKA (133x) - 57357: 4, // hintBNL (133x) - 57406: 5, // hintForceIndex (133x) - 57381: 6, // hintHashAgg (133x) - 57359: 7, // hintHashJoin (133x) - 57360: 8, // hintHashJoinBuild (133x) - 57361: 9, // hintHashJoinProbe (133x) - 57384: 10, // hintIgnoreIndex (133x) - 57380: 11, // hintIgnorePlanCache (133x) - 57365: 12, // hintIndexMerge (133x) - 57385: 13, // hintInlHashJoin (133x) - 57386: 14, // hintInlJoin (133x) - 57387: 15, // hintInlMergeJoin (133x) - 57351: 16, // hintJoinFixedOrder (133x) - 57352: 17, // hintJoinOrder (133x) - 57353: 18, // hintJoinPrefix (133x) - 57354: 19, // hintJoinSuffix (133x) - 57408: 20, // hintLeading (133x) - 57405: 21, // hintLimitToCop (133x) - 57375: 22, // hintMaxExecutionTime (133x) - 57388: 23, // hintMemoryQuota (133x) - 57363: 24, // hintMerge (133x) - 57382: 25, // hintMpp1PhaseAgg (133x) - 57383: 26, // hintMpp2PhaseAgg (133x) - 57367: 27, // hintMRR (133x) - 57356: 28, // hintNoBKA (133x) - 57358: 29, // hintNoBNL (133x) - 57410: 30, // hintNoDecorrelate (133x) - 57362: 31, // hintNoHashJoin (133x) - 57369: 32, // hintNoICP (133x) - 57366: 33, // hintNoIndexMerge (133x) - 57364: 34, // hintNoMerge (133x) - 57368: 35, // hintNoMRR (133x) - 57370: 36, // hintNoRangeOptimization (133x) - 57374: 37, // hintNoSemijoin (133x) - 57372: 38, // hintNoSkipScan (133x) - 57389: 39, // hintNoSwapJoinInputs (133x) - 57404: 40, // hintNthPlan (133x) - 57378: 41, // hintQBName (133x) - 57390: 42, // hintQueryType (133x) - 57391: 43, // hintReadConsistentReplica (133x) - 57392: 44, // hintReadFromStorage (133x) - 57377: 45, // hintResourceGroup (133x) - 57373: 46, // hintSemijoin (133x) - 57409: 47, // hintSemiJoinRewrite (133x) - 57376: 48, // hintSetVar (133x) - 57395: 49, // hintShuffleJoin (133x) - 57371: 50, // hintSkipScan (133x) - 57393: 51, // hintSMJoin (133x) - 57407: 52, // hintStraightJoin (133x) - 57396: 53, // hintStreamAgg (133x) - 57397: 54, // hintSwapJoinInputs (133x) - 57402: 55, // hintTimeRange (133x) - 57403: 56, // hintUseCascades (133x) - 57399: 57, // hintUseIndex (133x) - 57398: 58, // hintUseIndexMerge (133x) - 57400: 59, // hintUsePlanCache (133x) - 57401: 60, // hintUseToja (133x) - 44: 61, // ',' (130x) - 57420: 62, // hintDupsWeedOut (110x) - 57421: 63, // hintFirstMatch (110x) - 57422: 64, // hintLooseScan (110x) - 57423: 65, // hintMaterialization (110x) - 57415: 66, // hintTiFlash (110x) - 57414: 67, // hintTiKV (110x) - 57416: 68, // hintFalse (109x) - 57411: 69, // hintOLAP (109x) - 57412: 70, // hintOLTP (109x) - 57417: 71, // hintTrue (109x) - 57419: 72, // hintGB (108x) - 57418: 73, // hintMB (108x) - 57347: 74, // hintIdentifier (107x) - 57349: 75, // hintSingleAtIdentifier (93x) - 93: 76, // ']' (84x) - 46: 77, // '.' (83x) - 57413: 78, // hintPartition (78x) - 61: 79, // '=' (74x) - 40: 80, // '(' (69x) - 57344: 81, // $end (25x) - 57444: 82, // QueryBlockOpt (20x) - 57436: 83, // Identifier (15x) - 57346: 84, // hintIntLit (8x) - 57350: 85, // hintStringLit (5x) - 57426: 86, // CommaOpt (4x) - 57432: 87, // HintTable (4x) - 57433: 88, // HintTableList (4x) - 91: 89, // '[' (3x) - 57425: 90, // BooleanHintName (2x) - 57427: 91, // HintIndexList (2x) - 57429: 92, // HintStorageType (2x) - 57430: 93, // HintStorageTypeAndTable (2x) - 57434: 94, // HintTableListOpt (2x) - 57439: 95, // JoinOrderOptimizerHintName (2x) - 57440: 96, // NullaryHintName (2x) - 57443: 97, // PartitionListOpt (2x) - 57446: 98, // StorageOptimizerHintOpt (2x) - 57447: 99, // SubqueryOptimizerHintName (2x) - 57450: 100, // SubqueryStrategy (2x) - 57451: 101, // SupportedIndexLevelOptimizerHintName (2x) - 57452: 102, // SupportedTableLevelOptimizerHintName (2x) - 57453: 103, // TableOptimizerHintOpt (2x) - 57455: 104, // UnsupportedIndexLevelOptimizerHintName (2x) - 57456: 105, // UnsupportedTableLevelOptimizerHintName (2x) - 57458: 106, // ViewName (2x) - 57428: 107, // HintQueryType (1x) - 57431: 108, // HintStorageTypeAndTableList (1x) - 57435: 109, // HintTrueOrFalse (1x) - 57437: 110, // IndexNameList (1x) - 57438: 111, // IndexNameListOpt (1x) - 57441: 112, // OptimizerHintList (1x) - 57442: 113, // PartitionList (1x) - 57445: 114, // Start (1x) - 57448: 115, // SubqueryStrategies (1x) - 57449: 116, // SubqueryStrategiesOpt (1x) - 57454: 117, // UnitOfBytes (1x) - 57457: 118, // Value (1x) - 57459: 119, // ViewNameList (1x) - 57424: 120, // $default (0x) - 57345: 121, // error (0x) - 57348: 122, // hintInvalid (0x) + 41: 0, // ')' (148x) + 57379: 1, // hintAggToCop (135x) + 57394: 2, // hintBCJoin (135x) + 57355: 3, // hintBKA (135x) + 57357: 4, // hintBNL (135x) + 57408: 5, // hintForceIndex (135x) + 57381: 6, // hintHashAgg (135x) + 57359: 7, // hintHashJoin (135x) + 57360: 8, // hintHashJoinBuild (135x) + 57361: 9, // hintHashJoinProbe (135x) + 57384: 10, // hintIgnoreIndex (135x) + 57380: 11, // hintIgnorePlanCache (135x) + 57365: 12, // hintIndexMerge (135x) + 57385: 13, // hintInlHashJoin (135x) + 57386: 14, // hintInlJoin (135x) + 57387: 15, // hintInlMergeJoin (135x) + 57351: 16, // hintJoinFixedOrder (135x) + 57352: 17, // hintJoinOrder (135x) + 57353: 18, // hintJoinPrefix (135x) + 57354: 19, // hintJoinSuffix (135x) + 57400: 20, // hintKeepOrder (135x) + 57410: 21, // hintLeading (135x) + 57407: 22, // hintLimitToCop (135x) + 57375: 23, // hintMaxExecutionTime (135x) + 57388: 24, // hintMemoryQuota (135x) + 57363: 25, // hintMerge (135x) + 57382: 26, // hintMpp1PhaseAgg (135x) + 57383: 27, // hintMpp2PhaseAgg (135x) + 57367: 28, // hintMRR (135x) + 57356: 29, // hintNoBKA (135x) + 57358: 30, // hintNoBNL (135x) + 57412: 31, // hintNoDecorrelate (135x) + 57362: 32, // hintNoHashJoin (135x) + 57369: 33, // hintNoICP (135x) + 57366: 34, // hintNoIndexMerge (135x) + 57401: 35, // hintNoKeepOrder (135x) + 57364: 36, // hintNoMerge (135x) + 57368: 37, // hintNoMRR (135x) + 57370: 38, // hintNoRangeOptimization (135x) + 57374: 39, // hintNoSemijoin (135x) + 57372: 40, // hintNoSkipScan (135x) + 57389: 41, // hintNoSwapJoinInputs (135x) + 57406: 42, // hintNthPlan (135x) + 57378: 43, // hintQBName (135x) + 57390: 44, // hintQueryType (135x) + 57391: 45, // hintReadConsistentReplica (135x) + 57392: 46, // hintReadFromStorage (135x) + 57377: 47, // hintResourceGroup (135x) + 57373: 48, // hintSemijoin (135x) + 57411: 49, // hintSemiJoinRewrite (135x) + 57376: 50, // hintSetVar (135x) + 57395: 51, // hintShuffleJoin (135x) + 57371: 52, // hintSkipScan (135x) + 57393: 53, // hintSMJoin (135x) + 57409: 54, // hintStraightJoin (135x) + 57396: 55, // hintStreamAgg (135x) + 57397: 56, // hintSwapJoinInputs (135x) + 57404: 57, // hintTimeRange (135x) + 57405: 58, // hintUseCascades (135x) + 57399: 59, // hintUseIndex (135x) + 57398: 60, // hintUseIndexMerge (135x) + 57402: 61, // hintUsePlanCache (135x) + 57403: 62, // hintUseToja (135x) + 44: 63, // ',' (132x) + 57422: 64, // hintDupsWeedOut (112x) + 57423: 65, // hintFirstMatch (112x) + 57424: 66, // hintLooseScan (112x) + 57425: 67, // hintMaterialization (112x) + 57417: 68, // hintTiFlash (112x) + 57416: 69, // hintTiKV (112x) + 57418: 70, // hintFalse (111x) + 57413: 71, // hintOLAP (111x) + 57414: 72, // hintOLTP (111x) + 57419: 73, // hintTrue (111x) + 57421: 74, // hintGB (110x) + 57420: 75, // hintMB (110x) + 57347: 76, // hintIdentifier (109x) + 57349: 77, // hintSingleAtIdentifier (95x) + 93: 78, // ']' (86x) + 46: 79, // '.' (85x) + 57415: 80, // hintPartition (80x) + 61: 81, // '=' (76x) + 40: 82, // '(' (71x) + 57344: 83, // $end (25x) + 57446: 84, // QueryBlockOpt (20x) + 57438: 85, // Identifier (15x) + 57346: 86, // hintIntLit (8x) + 57350: 87, // hintStringLit (5x) + 57428: 88, // CommaOpt (4x) + 57434: 89, // HintTable (4x) + 57435: 90, // HintTableList (4x) + 91: 91, // '[' (3x) + 57427: 92, // BooleanHintName (2x) + 57429: 93, // HintIndexList (2x) + 57431: 94, // HintStorageType (2x) + 57432: 95, // HintStorageTypeAndTable (2x) + 57436: 96, // HintTableListOpt (2x) + 57441: 97, // JoinOrderOptimizerHintName (2x) + 57442: 98, // NullaryHintName (2x) + 57445: 99, // PartitionListOpt (2x) + 57448: 100, // StorageOptimizerHintOpt (2x) + 57449: 101, // SubqueryOptimizerHintName (2x) + 57452: 102, // SubqueryStrategy (2x) + 57453: 103, // SupportedIndexLevelOptimizerHintName (2x) + 57454: 104, // SupportedTableLevelOptimizerHintName (2x) + 57455: 105, // TableOptimizerHintOpt (2x) + 57457: 106, // UnsupportedIndexLevelOptimizerHintName (2x) + 57458: 107, // UnsupportedTableLevelOptimizerHintName (2x) + 57460: 108, // ViewName (2x) + 57430: 109, // HintQueryType (1x) + 57433: 110, // HintStorageTypeAndTableList (1x) + 57437: 111, // HintTrueOrFalse (1x) + 57439: 112, // IndexNameList (1x) + 57440: 113, // IndexNameListOpt (1x) + 57443: 114, // OptimizerHintList (1x) + 57444: 115, // PartitionList (1x) + 57447: 116, // Start (1x) + 57450: 117, // SubqueryStrategies (1x) + 57451: 118, // SubqueryStrategiesOpt (1x) + 57456: 119, // UnitOfBytes (1x) + 57459: 120, // Value (1x) + 57461: 121, // ViewNameList (1x) + 57426: 122, // $default (0x) + 57345: 123, // error (0x) + 57348: 124, // hintInvalid (0x) } yyhintSymNames = []string{ @@ -275,6 +279,7 @@ var ( "hintJoinOrder", "hintJoinPrefix", "hintJoinSuffix", + "hintKeepOrder", "hintLeading", "hintLimitToCop", "hintMaxExecutionTime", @@ -289,6 +294,7 @@ var ( "hintNoHashJoin", "hintNoICP", "hintNoIndexMerge", + "hintNoKeepOrder", "hintNoMerge", "hintNoMRR", "hintNoRangeOptimization", @@ -382,89 +388,76 @@ var ( yyhintReductions = []struct{ xsym, components int }{ {0, 1}, + {116, 1}, {114, 1}, - {112, 1}, - {112, 3}, - {112, 1}, - {112, 3}, - {103, 4}, - {103, 4}, - {103, 4}, - {103, 4}, - {103, 4}, - {103, 4}, - {103, 5}, - {103, 5}, - {103, 5}, - {103, 6}, - {103, 4}, - {103, 4}, - {103, 6}, - {103, 6}, - {103, 6}, - {103, 5}, - {103, 4}, - {103, 5}, - {98, 5}, - {108, 1}, - {108, 3}, - {93, 4}, - {82, 0}, - {82, 1}, - {86, 0}, - {86, 1}, - {97, 0}, - {97, 4}, - {113, 1}, - {113, 3}, - {94, 1}, - {94, 1}, - {88, 2}, - {88, 3}, - {87, 3}, - {87, 5}, - {119, 3}, - {119, 1}, - {106, 2}, - {106, 1}, - {91, 4}, - {111, 0}, - {111, 1}, + {114, 3}, + {114, 1}, + {114, 3}, + {105, 4}, + {105, 4}, + {105, 4}, + {105, 4}, + {105, 4}, + {105, 4}, + {105, 5}, + {105, 5}, + {105, 5}, + {105, 6}, + {105, 4}, + {105, 4}, + {105, 6}, + {105, 6}, + {105, 6}, + {105, 5}, + {105, 4}, + {105, 5}, + {100, 5}, {110, 1}, {110, 3}, - {116, 0}, - {116, 1}, + {95, 4}, + {84, 0}, + {84, 1}, + {88, 0}, + {88, 1}, + {99, 0}, + {99, 4}, {115, 1}, {115, 3}, + {96, 1}, + {96, 1}, + {90, 2}, + {90, 3}, + {89, 3}, + {89, 5}, + {121, 3}, + {121, 1}, + {108, 2}, + {108, 1}, + {93, 4}, + {113, 0}, + {113, 1}, + {112, 1}, + {112, 3}, + {118, 0}, {118, 1}, - {118, 1}, - {118, 1}, - {117, 1}, {117, 1}, - {109, 1}, - {109, 1}, - {95, 1}, - {95, 1}, - {95, 1}, - {105, 1}, - {105, 1}, - {105, 1}, - {105, 1}, - {105, 1}, - {105, 1}, - {102, 1}, - {102, 1}, - {102, 1}, - {102, 1}, - {102, 1}, - {102, 1}, - {102, 1}, - {102, 1}, - {102, 1}, - {102, 1}, - {102, 1}, - {102, 1}, - {102, 1}, + {117, 3}, + {120, 1}, + {120, 1}, + {120, 1}, + {119, 1}, + {119, 1}, + {111, 1}, + {111, 1}, + {97, 1}, + {97, 1}, + {97, 1}, + {107, 1}, + {107, 1}, + {107, 1}, + {107, 1}, + {107, 1}, + {107, 1}, {104, 1}, {104, 1}, {104, 1}, @@ -472,449 +465,471 @@ var ( {104, 1}, {104, 1}, {104, 1}, + {104, 1}, + {104, 1}, + {104, 1}, + {104, 1}, + {104, 1}, + {104, 1}, + {106, 1}, + {106, 1}, + {106, 1}, + {106, 1}, + {106, 1}, + {106, 1}, + {106, 1}, + {103, 1}, + {103, 1}, + {103, 1}, + {103, 1}, + {103, 1}, + {103, 1}, {101, 1}, {101, 1}, - {101, 1}, - {101, 1}, - {99, 1}, - {99, 1}, - {100, 1}, - {100, 1}, - {100, 1}, - {100, 1}, - {90, 1}, - {90, 1}, - {96, 1}, - {96, 1}, - {96, 1}, - {96, 1}, - {96, 1}, - {96, 1}, - {96, 1}, - {96, 1}, - {96, 1}, - {96, 1}, - {96, 1}, - {96, 1}, - {96, 1}, - {107, 1}, - {107, 1}, + {102, 1}, + {102, 1}, + {102, 1}, + {102, 1}, {92, 1}, {92, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, - {83, 1}, + {98, 1}, + {98, 1}, + {98, 1}, + {98, 1}, + {98, 1}, + {98, 1}, + {98, 1}, + {98, 1}, + {98, 1}, + {98, 1}, + {98, 1}, + {98, 1}, + {98, 1}, + {109, 1}, + {109, 1}, + {94, 1}, + {94, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, + {85, 1}, } yyhintXErrors = map[yyhintXError]string{} - yyhintParseTab = [280][]uint16{ + yyhintParseTab = [284][]uint16{ // 0 - {1: 258, 226, 219, 221, 248, 254, 234, 235, 236, 246, 262, 238, 230, 228, 233, 198, 216, 217, 218, 237, 259, 205, 210, 229, 255, 256, 239, 220, 222, 265, 223, 241, 260, 224, 240, 242, 250, 244, 232, 206, 209, 214, 261, 215, 208, 249, 264, 207, 227, 243, 225, 263, 257, 231, 211, 252, 245, 247, 253, 251, 90: 212, 95: 199, 213, 98: 197, 204, 101: 203, 201, 196, 202, 200, 112: 195, 114: 194}, - {81: 193}, - {1: 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 361, 81: 192, 86: 470}, - {1: 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 81: 191}, - {1: 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 81: 189}, + {1: 264, 230, 223, 225, 252, 260, 238, 239, 240, 250, 268, 242, 234, 232, 237, 202, 220, 221, 222, 253, 241, 265, 209, 214, 233, 261, 262, 243, 224, 226, 271, 227, 245, 266, 254, 228, 244, 246, 256, 248, 236, 210, 213, 218, 267, 219, 212, 255, 270, 211, 231, 247, 229, 269, 263, 235, 215, 258, 249, 251, 259, 257, 92: 216, 97: 203, 217, 100: 201, 208, 103: 207, 205, 200, 206, 204, 114: 199, 116: 198}, + {83: 197}, + {1: 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 369, 83: 196, 88: 478}, + {1: 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 83: 195}, + {1: 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 83: 193}, // 5 - {80: 467}, - {80: 464}, - {80: 461}, - {80: 456}, - {80: 453}, + {82: 475}, + {82: 472}, + {82: 469}, + {82: 464}, + {82: 461}, // 10 - {80: 442}, - {80: 430}, - {80: 426}, - {80: 422}, - {80: 414}, + {82: 450}, + {82: 438}, + {82: 434}, + {82: 430}, + {82: 422}, // 15 - {80: 411}, - {80: 399}, - {80: 392}, - {80: 387}, - {80: 381}, + {82: 419}, + {82: 407}, + {82: 400}, + {82: 395}, + {82: 389}, // 20 - {80: 378}, - {80: 372}, - {80: 266}, - {80: 131}, - {80: 130}, + {82: 386}, + {82: 380}, + {82: 272}, + {82: 135}, + {82: 134}, // 25 - {80: 129}, - {80: 128}, - {80: 127}, - {80: 126}, - {80: 125}, + {82: 133}, + {82: 132}, + {82: 131}, + {82: 130}, + {82: 129}, // 30 - {80: 124}, - {80: 123}, - {80: 122}, - {80: 121}, - {80: 120}, + {82: 128}, + {82: 127}, + {82: 126}, + {82: 125}, + {82: 124}, // 35 - {80: 119}, - {80: 118}, - {80: 117}, - {80: 116}, - {80: 115}, + {82: 123}, + {82: 122}, + {82: 121}, + {82: 120}, + {82: 119}, // 40 - {80: 114}, - {80: 113}, - {80: 112}, - {80: 111}, - {80: 110}, + {82: 118}, + {82: 117}, + {82: 116}, + {82: 115}, + {82: 114}, // 45 - {80: 109}, - {80: 108}, - {80: 107}, - {80: 106}, - {80: 105}, + {82: 113}, + {82: 112}, + {82: 111}, + {82: 110}, + {82: 109}, // 50 - {80: 104}, - {80: 103}, - {80: 102}, - {80: 101}, - {80: 100}, + {82: 108}, + {82: 107}, + {82: 106}, + {82: 105}, + {82: 104}, // 55 - {80: 99}, - {80: 98}, - {80: 97}, - {80: 92}, - {80: 91}, + {82: 103}, + {82: 102}, + {82: 101}, + {82: 100}, + {82: 99}, // 60 - {80: 90}, - {80: 89}, - {80: 88}, - {80: 87}, - {80: 86}, + {82: 94}, + {82: 93}, + {82: 92}, + {82: 91}, + {82: 90}, // 65 - {80: 85}, - {80: 84}, - {80: 83}, - {80: 82}, - {80: 81}, + {82: 89}, + {82: 88}, + {82: 87}, + {82: 86}, + {82: 85}, // 70 - {80: 80}, - {80: 79}, - {80: 78}, - {66: 165, 165, 75: 268, 82: 267}, - {66: 273, 272, 92: 271, 270, 108: 269}, + {82: 84}, + {82: 83}, + {82: 82}, + {82: 81}, + {82: 80}, // 75 - {164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 76: 164, 164, 164, 84: 164}, - {369, 61: 370}, - {168, 61: 168}, - {89: 274}, - {89: 75}, + {68: 169, 169, 77: 274, 84: 273}, + {68: 279, 278, 94: 277, 276, 110: 275}, + {168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 78: 168, 168, 168, 86: 168}, + {377, 63: 378}, + {172, 63: 172}, // 80 - {89: 74}, - {1: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 276, 88: 275}, - {61: 367, 76: 366}, - {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 278, 87: 277}, - {155, 61: 155, 76: 155}, + {91: 280}, + {91: 77}, + {91: 76}, + {1: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 282, 90: 281}, + {63: 375, 78: 374}, // 85 - {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 165, 353, 165, 82: 352}, - {73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73}, - {72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72}, - {71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71}, - {70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70}, + {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 284, 89: 283}, + {159, 63: 159, 78: 159}, + {169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 169, 361, 169, 84: 360}, + {75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75}, + {74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74}, // 90 - {69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69}, - {68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68}, - {67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67}, - {66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66}, - {65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65}, + {73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73}, + {72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72}, + {71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71}, + {70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70}, + {69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69}, // 95 - {64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64}, - {63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63}, - {62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62}, - {61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61}, - {60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60}, + {68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68}, + {67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67}, + {66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66}, + {65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65}, + {64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64}, // 100 - {59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59}, - {58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58}, - {57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57}, - {56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56}, - {55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55}, + {63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63}, + {62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62}, + {61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61}, + {60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60}, + {59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59}, // 105 - {54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54}, - {53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53}, - {52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52}, - {51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51}, - {50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50}, + {58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58}, + {57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57}, + {56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56}, + {55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55}, + {54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54}, // 110 - {49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49}, - {48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48}, - {47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47}, - {46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46}, - {45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45}, + {53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53}, + {52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52}, + {51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51}, + {50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50}, + {49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49}, // 115 - {44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44}, - {43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43}, - {42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42}, - {41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41}, - {40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40}, + {48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48}, + {47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47}, + {46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46}, + {45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45}, + {44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44}, // 120 - {39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39}, - {38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38}, - {37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37}, - {36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36}, - {35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35}, + {43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43}, + {42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42}, + {41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41}, + {40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40}, + {39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39}, // 125 - {34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34}, - {33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33}, - {32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32}, - {31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31}, - {30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30}, + {38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38}, + {37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37}, + {36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36}, + {35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35}, + {34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34}, // 130 - {29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29}, - {28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28}, - {27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27}, - {26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26}, - {25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25}, + {33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33}, + {32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32}, + {31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31}, + {30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30}, + {29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29}, // 135 - {24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24}, - {23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23}, - {22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22}, - {21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21}, - {20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20}, + {28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28}, + {27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27}, + {26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26}, + {25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25}, + {24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24}, // 140 - {19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19}, - {18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18}, - {17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17}, - {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16}, - {15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}, + {23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23}, + {22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22}, + {21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21}, + {20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20}, + {19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19}, // 145 - {14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14}, - {13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13}, - {12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12}, - {11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11}, - {10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10}, + {18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18}, + {17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17}, + {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16}, + {15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}, + {14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14}, // 150 - {9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9}, - {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8}, - {7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7}, - {6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6}, - {5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5}, + {13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13}, + {12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12}, + {11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11}, + {10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10}, + {9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9}, // 155 - {4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4}, - {3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3}, - {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, - {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, - {161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 76: 161, 78: 356, 97: 365}, + {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8}, + {7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7}, + {6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6}, + {5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5}, + {4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4}, // 160 - {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 354}, - {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 165, 78: 165, 82: 355}, - {161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 76: 161, 78: 356, 97: 357}, - {80: 358}, - {152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 76: 152}, + {3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3}, + {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, + {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 78: 165, 80: 364, 99: 373}, + {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 362}, // 165 - {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 360, 113: 359}, - {362, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 361, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 86: 363}, - {159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159}, - {162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 62: 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 85: 162}, - {160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 76: 160}, + {169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 169, 80: 169, 84: 363}, + {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 78: 165, 80: 364, 99: 365}, + {82: 366}, + {156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 78: 156}, + {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 368, 115: 367}, // 170 - {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 364}, - {158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158}, - {153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 76: 153}, - {166, 61: 166}, - {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 278, 87: 368}, + {370, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 369, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 88: 371}, + {163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163}, + {166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 64: 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 87: 166}, + {164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 78: 164}, + {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 372}, // 175 - {154, 61: 154, 76: 154}, - {1: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 81: 169}, - {66: 273, 272, 92: 271, 371}, - {167, 61: 167}, - {69: 165, 165, 75: 268, 82: 373}, + {162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162}, + {157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 78: 157}, + {170, 63: 170}, + {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 284, 89: 376}, + {158, 63: 158, 78: 158}, // 180 - {69: 375, 376, 107: 374}, - {377}, - {77}, - {76}, - {1: 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 81: 170}, + {1: 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 83: 173}, + {68: 279, 278, 94: 277, 379}, + {171, 63: 171}, + {71: 169, 169, 77: 274, 84: 381}, + {71: 383, 384, 109: 382}, // 185 - {165, 75: 268, 82: 379}, - {380}, - {1: 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 81: 171}, - {68: 165, 71: 165, 75: 268, 82: 382}, - {68: 385, 71: 384, 109: 383}, + {385}, + {79}, + {78}, + {1: 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 83: 174}, + {169, 77: 274, 84: 387}, // 190 - {386}, - {133}, - {132}, - {1: 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 81: 172}, - {85: 388}, + {388}, + {1: 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 83: 175}, + {70: 169, 73: 169, 77: 274, 84: 390}, + {70: 393, 73: 392, 111: 391}, + {394}, // 195 - {61: 361, 85: 163, 389}, - {85: 390}, - {391}, - {1: 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 81: 173}, - {75: 268, 82: 393, 84: 165}, + {137}, + {136}, + {1: 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 83: 176}, + {87: 396}, + {63: 369, 87: 167, 397}, // 200 - {84: 394}, - {72: 397, 396, 117: 395}, - {398}, - {135}, - {134}, + {87: 398}, + {399}, + {1: 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 83: 177}, + {77: 274, 84: 401, 86: 169}, + {86: 402}, // 205 - {1: 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 81: 174}, - {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 400}, - {401, 61: 402}, - {1: 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 81: 176}, - {165, 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 268, 77: 165, 82: 406, 405, 106: 404, 119: 403}, + {74: 405, 404, 119: 403}, + {406}, + {139}, + {138}, + {1: 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 83: 178}, // 210 - {408, 77: 409}, - {150, 77: 150}, - {165, 75: 268, 77: 165, 82: 407}, - {148, 77: 148}, - {149, 77: 149}, + {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 408}, + {409, 63: 410}, + {1: 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 83: 180}, + {169, 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 274, 79: 169, 84: 414, 413, 108: 412, 121: 411}, + {416, 79: 417}, // 215 - {1: 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 81: 175}, - {165, 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 268, 77: 165, 82: 406, 405, 106: 410}, - {151, 77: 151}, - {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 412}, - {413}, + {154, 79: 154}, + {169, 77: 274, 79: 169, 84: 415}, + {152, 79: 152}, + {153, 79: 153}, + {1: 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 83: 179}, // 220 - {1: 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 81: 177}, - {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 415}, - {79: 416}, - {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 419, 420, 418, 118: 417}, + {169, 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 274, 79: 169, 84: 414, 413, 108: 418}, + {155, 79: 155}, + {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 420}, {421}, + {1: 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 83: 181}, // 225 - {138}, - {137}, - {136}, - {1: 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 81: 178}, - {75: 268, 82: 423, 84: 165}, + {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 423}, + {81: 424}, + {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 427, 428, 426, 120: 425}, + {429}, + {142}, // 230 - {84: 424}, - {425}, - {1: 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 81: 179}, - {75: 268, 82: 427, 84: 165}, - {84: 428}, + {141}, + {140}, + {1: 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 83: 182}, + {77: 274, 84: 431, 86: 169}, + {86: 432}, // 235 - {429}, - {1: 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 81: 180}, - {165, 62: 165, 165, 165, 165, 75: 268, 82: 431}, - {142, 62: 435, 436, 437, 438, 100: 434, 115: 433, 432}, - {441}, + {433}, + {1: 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 83: 183}, + {77: 274, 84: 435, 86: 169}, + {86: 436}, + {437}, // 240 - {141, 61: 439}, - {140, 61: 140}, - {96, 61: 96}, - {95, 61: 95}, - {94, 61: 94}, + {1: 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 83: 184}, + {169, 64: 169, 169, 169, 169, 77: 274, 84: 439}, + {146, 64: 443, 444, 445, 446, 102: 442, 117: 441, 440}, + {449}, + {145, 63: 447}, // 245 - {93, 61: 93}, - {62: 435, 436, 437, 438, 100: 440}, - {139, 61: 139}, - {1: 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 81: 181}, - {1: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 444, 91: 443}, + {144, 63: 144}, + {98, 63: 98}, + {97, 63: 97}, + {96, 63: 96}, + {95, 63: 95}, // 250 - {452}, - {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 278, 87: 445}, - {163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 361, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 86: 446}, - {146, 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 449, 110: 448, 447}, - {147}, + {64: 443, 444, 445, 446, 102: 448}, + {143, 63: 143}, + {1: 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 83: 185}, + {1: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 452, 93: 451}, + {460}, // 255 - {145, 61: 450}, - {144, 61: 144}, - {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 451}, - {143, 61: 143}, - {1: 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 81: 182}, + {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 284, 89: 453}, + {167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 369, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 88: 454}, + {150, 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 457, 112: 456, 455}, + {151}, + {149, 63: 458}, // 260 - {1: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 444, 91: 454}, - {455}, - {1: 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 81: 183}, - {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 459, 88: 458, 94: 457}, - {460}, + {148, 63: 148}, + {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 459}, + {147, 63: 147}, + {1: 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 83: 186}, + {1: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 452, 93: 462}, // 265 - {157, 61: 367}, - {156, 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 278, 87: 277}, - {1: 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 81: 184}, - {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 459, 88: 458, 94: 462}, {463}, + {1: 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 83: 187}, + {169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 467, 90: 466, 96: 465}, + {468}, + {161, 63: 375}, // 270 - {1: 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 81: 185}, - {1: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 276, 88: 465}, - {466, 61: 367}, - {1: 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 81: 186}, - {165, 75: 268, 82: 468}, + {160, 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 336, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 337, 299, 303, 305, 309, 307, 325, 342, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 284, 89: 283}, + {1: 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 83: 188}, + {169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 467, 90: 466, 96: 470}, + {471}, + {1: 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 83: 189}, // 275 - {469}, - {1: 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 81: 187}, - {1: 258, 226, 219, 221, 248, 254, 234, 235, 236, 246, 262, 238, 230, 228, 233, 198, 216, 217, 218, 237, 259, 205, 210, 229, 255, 256, 239, 220, 222, 265, 223, 241, 260, 224, 240, 242, 250, 244, 232, 206, 209, 214, 261, 215, 208, 249, 264, 207, 227, 243, 225, 263, 257, 231, 211, 252, 245, 247, 253, 251, 90: 212, 95: 199, 213, 98: 472, 204, 101: 203, 201, 471, 202, 200}, - {1: 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 81: 190}, - {1: 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 81: 188}, + {1: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 282, 90: 473}, + {474, 63: 375}, + {1: 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 83: 190}, + {169, 77: 274, 84: 476}, + {477}, + // 280 + {1: 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 83: 191}, + {1: 264, 230, 223, 225, 252, 260, 238, 239, 240, 250, 268, 242, 234, 232, 237, 202, 220, 221, 222, 253, 241, 265, 209, 214, 233, 261, 262, 243, 224, 226, 271, 227, 245, 266, 254, 228, 244, 246, 256, 248, 236, 210, 213, 218, 267, 219, 212, 255, 270, 211, 231, 247, 229, 269, 263, 235, 215, 258, 249, 251, 259, 257, 92: 216, 97: 203, 217, 100: 480, 208, 103: 207, 205, 479, 206, 204}, + {1: 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 83: 194}, + {1: 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 83: 192}, } ) @@ -954,7 +969,7 @@ func yyhintlex1(yylex yyhintLexer, lval *yyhintSymType) (n int) { } func yyhintParse(yylex yyhintLexer, parser *hintParser) int { - const yyError = 121 + const yyError = 123 yyEx, _ := yylex.(yyhintLexerEx) var yyn int diff --git a/parser/hintparser.y b/parser/hintparser.y index a2ccd21d2d401..0d92905bb86f3 100644 --- a/parser/hintparser.y +++ b/parser/hintparser.y @@ -103,6 +103,8 @@ import ( hintSwapJoinInputs "SWAP_JOIN_INPUTS" hintUseIndexMerge "USE_INDEX_MERGE" hintUseIndex "USE_INDEX" + hintKeepOrder "KEEP_ORDER" + hintNoKeepOrder "NO_KEEP_ORDER" hintUsePlanCache "USE_PLAN_CACHE" hintUseToja "USE_TOJA" hintTimeRange "TIME_RANGE" @@ -607,6 +609,8 @@ SupportedIndexLevelOptimizerHintName: | "IGNORE_INDEX" | "USE_INDEX_MERGE" | "FORCE_INDEX" +| "KEEP_ORDER" +| "NO_KEEP_ORDER" SubqueryOptimizerHintName: "SEMIJOIN" @@ -699,6 +703,8 @@ Identifier: | "SWAP_JOIN_INPUTS" | "USE_INDEX_MERGE" | "USE_INDEX" +| "KEEP_ORDER" +| "NO_KEEP_ORDER" | "USE_PLAN_CACHE" | "USE_TOJA" | "TIME_RANGE" diff --git a/parser/misc.go b/parser/misc.go index 3c859c30283b1..c5bda32024409 100644 --- a/parser/misc.go +++ b/parser/misc.go @@ -956,6 +956,8 @@ var hintTokenMap = map[string]int{ "SWAP_JOIN_INPUTS": hintSwapJoinInputs, "USE_INDEX_MERGE": hintUseIndexMerge, "USE_INDEX": hintUseIndex, + "KEEP_ORDER": hintKeepOrder, + "NO_KEEP_ORDER": hintNoKeepOrder, "USE_PLAN_CACHE": hintUsePlanCache, "USE_TOJA": hintUseToja, "TIME_RANGE": hintTimeRange, diff --git a/parser/model/ddl.go b/parser/model/ddl.go index 7bb9eaef01e6f..c278ffb56167d 100644 --- a/parser/model/ddl.go +++ b/parser/model/ddl.go @@ -316,12 +316,19 @@ type MultiSchemaInfo struct { AddIndexes []CIStr `json:"-"` DropIndexes []CIStr `json:"-"` AlterIndexes []CIStr `json:"-"` - ForeignKeys []CIStr `json:"-"` + + AddForeignKeys []AddForeignKeyInfo `json:"-"` RelativeColumns []CIStr `json:"-"` PositionColumns []CIStr `json:"-"` } +// AddForeignKeyInfo contains foreign key information. +type AddForeignKeyInfo struct { + Name CIStr + Cols []CIStr +} + // NewMultiSchemaInfo new a MultiSchemaInfo. func NewMultiSchemaInfo() *MultiSchemaInfo { return &MultiSchemaInfo{ diff --git a/parser/model/model.go b/parser/model/model.go index ba7c46bcd6333..411db3fcf1d20 100644 --- a/parser/model/model.go +++ b/parser/model/model.go @@ -381,8 +381,8 @@ func FindFKInfoByName(fks []*FKInfo, name string) *FKInfo { } // FindIndexByColumns find IndexInfo in indices which is cover the specified columns. -func FindIndexByColumns(tbInfo *TableInfo, cols ...CIStr) *IndexInfo { - for _, index := range tbInfo.Indices { +func FindIndexByColumns(tbInfo *TableInfo, indices []*IndexInfo, cols ...CIStr) *IndexInfo { + for _, index := range indices { if IsIndexPrefixCovered(tbInfo, index, cols...) { return index } @@ -1419,6 +1419,7 @@ type IndexInfo struct { Primary bool `json:"is_primary"` // Whether the index is primary key. Invisible bool `json:"is_invisible"` // Whether the index is invisible. Global bool `json:"is_global"` // Whether the index is global. + MVIndex bool `json:"mv_index"` // Whether the index is multivalued index. } // Clone clones IndexInfo. diff --git a/parser/parser_test.go b/parser/parser_test.go index c06d2076f085a..dae08a80afde3 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -1603,6 +1603,7 @@ func TestBuiltin(t *testing.T) { {"select cast(time '2000' as year);", true, "SELECT CAST(TIME '2000' AS YEAR)"}, {"select cast(b as signed array);", true, "SELECT CAST(`b` AS SIGNED ARRAY)"}, + {"select cast(b as char(10) array);", true, "SELECT CAST(`b` AS CHAR(10) ARRAY)"}, // for last_insert_id {"SELECT last_insert_id();", true, "SELECT LAST_INSERT_ID()"}, @@ -3243,6 +3244,7 @@ func TestDDL(t *testing.T) { {"create table t (a bigint, b bigint as (a+1) not null);", true, "CREATE TABLE `t` (`a` BIGINT,`b` BIGINT GENERATED ALWAYS AS(`a`+1) VIRTUAL NOT NULL)"}, {"create table t (a bigint, b bigint as (a+1) not null);", true, "CREATE TABLE `t` (`a` BIGINT,`b` BIGINT GENERATED ALWAYS AS(`a`+1) VIRTUAL NOT NULL)"}, {"create table t (a bigint, b bigint as (a+1) not null comment 'ttt');", true, "CREATE TABLE `t` (`a` BIGINT,`b` BIGINT GENERATED ALWAYS AS(`a`+1) VIRTUAL NOT NULL COMMENT 'ttt')"}, + {"create table t(a int, index idx((cast(a as binary(1)))));", true, "CREATE TABLE `t` (`a` INT,INDEX `idx`((CAST(`a` AS BINARY(1)))))"}, {"alter table t add column (f timestamp as (a+1) default '2019-01-01 11:11:11');", false, ""}, {"alter table t modify column f int as (a+1) default 55;", false, ""}, @@ -3890,6 +3892,44 @@ func TestOptimizerHints(t *testing.T) { require.Len(t, hints[1].Indexes, 1) require.Equal(t, "t4", hints[1].Indexes[0].L) + // Test KEEP_ORDER + stmt, _, err = p.Parse("select /*+ KEEP_ORDER(T1,T2), keep_order(t3,t4) */ c1, c2 from t1, t2 where t1.c1 = t2.c1", "", "") + require.NoError(t, err) + selectStmt = stmt[0].(*ast.SelectStmt) + + hints = selectStmt.TableHints + require.Len(t, hints, 2) + require.Equal(t, "keep_order", hints[0].HintName.L) + require.Len(t, hints[0].Tables, 1) + require.Equal(t, "t1", hints[0].Tables[0].TableName.L) + require.Len(t, hints[0].Indexes, 1) + require.Equal(t, "t2", hints[0].Indexes[0].L) + + require.Equal(t, "keep_order", hints[1].HintName.L) + require.Len(t, hints[1].Tables, 1) + require.Equal(t, "t3", hints[1].Tables[0].TableName.L) + require.Len(t, hints[1].Indexes, 1) + require.Equal(t, "t4", hints[1].Indexes[0].L) + + // Test NO_KEEP_ORDER + stmt, _, err = p.Parse("select /*+ NO_KEEP_ORDER(T1,T2), no_keep_order(t3,t4) */ c1, c2 from t1, t2 where t1.c1 = t2.c1", "", "") + require.NoError(t, err) + selectStmt = stmt[0].(*ast.SelectStmt) + + hints = selectStmt.TableHints + require.Len(t, hints, 2) + require.Equal(t, "no_keep_order", hints[0].HintName.L) + require.Len(t, hints[0].Tables, 1) + require.Equal(t, "t1", hints[0].Tables[0].TableName.L) + require.Len(t, hints[0].Indexes, 1) + require.Equal(t, "t2", hints[0].Indexes[0].L) + + require.Equal(t, "no_keep_order", hints[1].HintName.L) + require.Len(t, hints[1].Tables, 1) + require.Equal(t, "t3", hints[1].Tables[0].TableName.L) + require.Len(t, hints[1].Indexes, 1) + require.Equal(t, "t4", hints[1].Indexes[0].L) + // Test TIDB_SMJ stmt, _, err = p.Parse("select /*+ TIDB_SMJ(T1,t2), tidb_smj(T3,t4) */ c1, c2 from t1, t2 where t1.c1 = t2.c1", "", "") require.NoError(t, err) diff --git a/parser/types/field_type.go b/parser/types/field_type.go index 369ed59fa7a59..991dc3d087d75 100644 --- a/parser/types/field_type.go +++ b/parser/types/field_type.go @@ -72,7 +72,7 @@ func NewFieldType(tp byte) *FieldType { // IsDecimalValid checks whether the decimal is valid. func (ft *FieldType) IsDecimalValid() bool { - if ft.tp == mysql.TypeNewDecimal && (ft.decimal < 0 || ft.decimal > mysql.MaxDecimalScale || ft.flen <= 0 || ft.flen > mysql.MaxDecimalWidth || ft.flen < ft.decimal) { + if ft.GetType() == mysql.TypeNewDecimal && (ft.decimal < 0 || ft.decimal > mysql.MaxDecimalScale || ft.flen <= 0 || ft.flen > mysql.MaxDecimalWidth || ft.flen < ft.decimal) { return false } return true @@ -80,7 +80,7 @@ func (ft *FieldType) IsDecimalValid() bool { // IsVarLengthType Determine whether the column type is a variable-length type func (ft *FieldType) IsVarLengthType() bool { - switch ft.tp { + switch ft.GetType() { case mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeJSON, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: return true default: @@ -90,6 +90,9 @@ func (ft *FieldType) IsVarLengthType() bool { // GetType returns the type of the FieldType. func (ft *FieldType) GetType() byte { + if ft.array { + return mysql.TypeJSON + } return ft.tp } @@ -126,6 +129,7 @@ func (ft *FieldType) GetElems() []string { // SetType sets the type of the FieldType. func (ft *FieldType) SetType(tp byte) { ft.tp = tp + ft.array = false } // SetFlag sets the flag of the FieldType. @@ -160,7 +164,7 @@ func (ft *FieldType) SetFlen(flen int) { // SetFlenUnderLimit sets the length of the field to the value of the argument func (ft *FieldType) SetFlenUnderLimit(flen int) { - if ft.tp == mysql.TypeNewDecimal { + if ft.GetType() == mysql.TypeNewDecimal { ft.flen = mathutil.Min(flen, mysql.MaxDecimalWidth) } else { ft.flen = flen @@ -174,7 +178,7 @@ func (ft *FieldType) SetDecimal(decimal int) { // SetDecimalUnderLimit sets the decimal of the field to the value of the argument func (ft *FieldType) SetDecimalUnderLimit(decimal int) { - if ft.tp == mysql.TypeNewDecimal { + if ft.GetType() == mysql.TypeNewDecimal { ft.decimal = mathutil.Min(decimal, mysql.MaxDecimalScale) } else { ft.decimal = decimal @@ -183,7 +187,7 @@ func (ft *FieldType) SetDecimalUnderLimit(decimal int) { // UpdateFlenAndDecimalUnderLimit updates the length and decimal to the value of the argument func (ft *FieldType) UpdateFlenAndDecimalUnderLimit(old *FieldType, deltaDecimal int, deltaFlen int) { - if ft.tp != mysql.TypeNewDecimal { + if ft.GetType() != mysql.TypeNewDecimal { return } if old.decimal < 0 { @@ -229,6 +233,13 @@ func (ft *FieldType) IsArray() bool { return ft.array } +// ArrayType return the type of the array. +func (ft *FieldType) ArrayType() *FieldType { + clone := ft.Clone() + clone.SetArray(false) + return clone +} + // SetElemWithIsBinaryLit sets the element of the FieldType. func (ft *FieldType) SetElemWithIsBinaryLit(idx int, element string, isBinaryLit bool) { ft.elems[idx] = element @@ -274,7 +285,7 @@ func (ft *FieldType) Equal(other *FieldType) bool { // When tp is float or double with decimal unspecified, do not check whether flen is equal, // because flen for them is useless. // The decimal field can be ignored if the type is int or string. - tpEqual := (ft.tp == other.tp) || (ft.tp == mysql.TypeVarchar && other.tp == mysql.TypeVarString) || (ft.tp == mysql.TypeVarString && other.tp == mysql.TypeVarchar) + tpEqual := (ft.GetType() == other.GetType()) || (ft.GetType() == mysql.TypeVarchar && other.GetType() == mysql.TypeVarString) || (ft.GetType() == mysql.TypeVarString && other.GetType() == mysql.TypeVarchar) flenEqual := ft.flen == other.flen || (ft.EvalType() == ETReal && ft.decimal == UnspecifiedLength) ignoreDecimal := ft.EvalType() == ETInt || ft.EvalType() == ETString partialEqual := tpEqual && @@ -316,7 +327,7 @@ func (ft *FieldType) PartialEqual(other *FieldType, unsafe bool) bool { // EvalType gets the type in evaluation. func (ft *FieldType) EvalType() EvalType { - switch ft.tp { + switch ft.GetType() { case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeBit, mysql.TypeYear: return ETInt @@ -342,7 +353,7 @@ func (ft *FieldType) EvalType() EvalType { // Hybrid checks whether a type is a hybrid type, which can represent different types of value in specific context. func (ft *FieldType) Hybrid() bool { - return ft.tp == mysql.TypeEnum || ft.tp == mysql.TypeBit || ft.tp == mysql.TypeSet + return ft.GetType() == mysql.TypeEnum || ft.GetType() == mysql.TypeBit || ft.GetType() == mysql.TypeSet } // Init initializes the FieldType data. @@ -355,10 +366,10 @@ func (ft *FieldType) Init(tp byte) { // CompactStr only considers tp/CharsetBin/flen/Deimal. // This is used for showing column type in infoschema. func (ft *FieldType) CompactStr() string { - ts := TypeToStr(ft.tp, ft.charset) + ts := TypeToStr(ft.GetType(), ft.charset) suffix := "" - defaultFlen, defaultDecimal := mysql.GetDefaultFieldLengthAndDecimal(ft.tp) + defaultFlen, defaultDecimal := mysql.GetDefaultFieldLengthAndDecimal(ft.GetType()) isDecimalNotDefault := ft.decimal != defaultDecimal && ft.decimal != 0 && ft.decimal != UnspecifiedLength // displayFlen and displayDecimal are flen and decimal values with `-1` substituted with default value. @@ -370,7 +381,7 @@ func (ft *FieldType) CompactStr() string { displayDecimal = defaultDecimal } - switch ft.tp { + switch ft.GetType() { case mysql.TypeEnum, mysql.TypeSet: // Format is ENUM ('e1', 'e2') or SET ('e1', 'e2') es := make([]string, 0, len(ft.elems)) @@ -414,8 +425,8 @@ func (ft *FieldType) CompactStr() string { func (ft *FieldType) InfoSchemaStr() string { suffix := "" if mysql.HasUnsignedFlag(ft.flag) && - ft.tp != mysql.TypeBit && - ft.tp != mysql.TypeYear { + ft.GetType() != mysql.TypeBit && + ft.GetType() != mysql.TypeYear { suffix = " unsigned" } return ft.CompactStr() + suffix @@ -431,11 +442,11 @@ func (ft *FieldType) String() string { if mysql.HasZerofillFlag(ft.flag) { strs = append(strs, "ZEROFILL") } - if mysql.HasBinaryFlag(ft.flag) && ft.tp != mysql.TypeString { + if mysql.HasBinaryFlag(ft.flag) && ft.GetType() != mysql.TypeString { strs = append(strs, "BINARY") } - if IsTypeChar(ft.tp) || IsTypeBlob(ft.tp) { + if IsTypeChar(ft.GetType()) || IsTypeBlob(ft.GetType()) { if ft.charset != "" && ft.charset != charset.CharsetBin { strs = append(strs, fmt.Sprintf("CHARACTER SET %s", ft.charset)) } @@ -449,12 +460,12 @@ func (ft *FieldType) String() string { // Restore implements Node interface. func (ft *FieldType) Restore(ctx *format.RestoreCtx) error { - ctx.WriteKeyWord(TypeToStr(ft.tp, ft.charset)) + ctx.WriteKeyWord(TypeToStr(ft.GetType(), ft.charset)) precision := UnspecifiedLength scale := UnspecifiedLength - switch ft.tp { + switch ft.GetType() { case mysql.TypeEnum, mysql.TypeSet: ctx.WritePlain("(") for i, e := range ft.elems { @@ -491,7 +502,7 @@ func (ft *FieldType) Restore(ctx *format.RestoreCtx) error { ctx.WriteKeyWord(" BINARY") } - if IsTypeChar(ft.tp) || IsTypeBlob(ft.tp) { + if IsTypeChar(ft.GetType()) || IsTypeBlob(ft.GetType()) { if ft.charset != "" && ft.charset != charset.CharsetBin { ctx.WriteKeyWord(" CHARACTER SET " + ft.charset) } @@ -507,7 +518,7 @@ func (ft *FieldType) Restore(ctx *format.RestoreCtx) error { // RestoreAsCastType is used for write AST back to string. func (ft *FieldType) RestoreAsCastType(ctx *format.RestoreCtx, explicitCharset bool) { switch ft.tp { - case mysql.TypeVarString: + case mysql.TypeVarString, mysql.TypeString: skipWriteBinary := false if ft.charset == charset.CharsetBin && ft.collate == charset.CollationBin { ctx.WriteKeyWord("BINARY") @@ -519,7 +530,7 @@ func (ft *FieldType) RestoreAsCastType(ctx *format.RestoreCtx, explicitCharset b ctx.WritePlainf("(%d)", ft.flen) } if !explicitCharset { - return + break } if !skipWriteBinary && ft.flag&mysql.BinaryFlag != 0 { ctx.WriteKeyWord(" BINARY") @@ -581,7 +592,7 @@ const VarStorageLen = -1 // StorageLength is the length of stored value for the type. func (ft *FieldType) StorageLength() int { - switch ft.tp { + switch ft.GetType() { case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeDouble, mysql.TypeFloat, mysql.TypeYear, mysql.TypeDuration, mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp, mysql.TypeEnum, mysql.TypeSet, @@ -599,7 +610,7 @@ func (ft *FieldType) StorageLength() int { // HasCharset indicates if a COLUMN has an associated charset. Returning false here prevents some information // statements(like `SHOW CREATE TABLE`) from attaching a CHARACTER SET clause to the column. func HasCharset(ft *FieldType) bool { - switch ft.tp { + switch ft.GetType() { case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: return !mysql.HasBinaryFlag(ft.flag) diff --git a/planner/core/BUILD.bazel b/planner/core/BUILD.bazel index 0495f1f50cae0..1a7ec1196f0f1 100644 --- a/planner/core/BUILD.bazel +++ b/planner/core/BUILD.bazel @@ -18,6 +18,7 @@ go_library( "handle_cols.go", "hashcode.go", "hints.go", + "indexmerge_path.go", "initialize.go", "logical_plan_builder.go", "logical_plans.go", diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index f11f60b95cfe5..53aa657d2f9c0 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -252,6 +252,8 @@ const ( OpSetBindingStatus // OpSQLBindDropByDigest is used to drop SQL binds by digest OpSQLBindDropByDigest + // OpSetBindingStatusByDigest represents the operation to set SQL binding status by sql digest. + OpSetBindingStatusByDigest ) // SQLBindPlan represents a plan for SQL bind. @@ -914,11 +916,6 @@ func (e *Explain) explainOpRecursivelyInJSONFormat(flatOp *FlatOperator, flats F textTreeExplainID := texttree.PrettyIdentifier(explainID, flatOp.TextTreeIndent, flatOp.IsLastChild) cur := e.prepareOperatorInfoForJSONFormat(flatOp.Origin, taskTp, textTreeExplainID, explainID) - if e.ctx != nil && e.ctx.GetSessionVars() != nil && e.ctx.GetSessionVars().StmtCtx != nil { - if optimInfo, ok := e.ctx.GetSessionVars().StmtCtx.OptimInfo[flatOp.Origin.ID()]; ok { - e.ctx.GetSessionVars().StmtCtx.AppendNote(errors.New(optimInfo)) - } - } for _, idx := range flatOp.ChildrenIdx { cur.SubOperators = append(cur.SubOperators, @@ -938,11 +935,6 @@ func (e *Explain) explainFlatOpInRowFormat(flatOp *FlatOperator) { flatOp.TextTreeIndent, flatOp.IsLastChild) e.prepareOperatorInfo(flatOp.Origin, taskTp, textTreeExplainID) - if e.ctx != nil && e.ctx.GetSessionVars() != nil && e.ctx.GetSessionVars().StmtCtx != nil { - if optimInfo, ok := e.ctx.GetSessionVars().StmtCtx.OptimInfo[flatOp.Origin.ID()]; ok { - e.ctx.GetSessionVars().StmtCtx.AppendNote(errors.New(optimInfo)) - } - } } func getRuntimeInfoStr(ctx sessionctx.Context, p Plan, runtimeStatsColl *execdetails.RuntimeStatsColl) (actRows, analyzeInfo, memoryInfo, diskInfo string) { diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 768a8c20fc0b5..dfd8d9572db81 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -1928,7 +1928,7 @@ func (p *LogicalJoin) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]P } }) - if (p.preferJoinType&preferBCJoin) == 0 && p.preferJoinType > 0 { + if (p.preferJoinType&preferBCJoin) == 0 && (p.preferJoinType&preferShuffleJoin) == 0 && p.preferJoinType > 0 { p.SCtx().GetSessionVars().RaiseWarningWhenMPPEnforced("MPP mode may be blocked because you have used hint to specify a join algorithm which is not supported by mpp now.") if prop.IsFlashProp() { return nil, false, nil @@ -1957,6 +1957,21 @@ func (p *LogicalJoin) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]P mppJoins := p.tryToGetMppHashJoin(prop, false) joins = append(joins, mppJoins...) } + } else { + hasMppHints := false + var errMsg string + if (p.preferJoinType & preferShuffleJoin) > 0 { + errMsg = "The join can not push down to the MPP side, the shuffle_join() hint is invalid" + hasMppHints = true + } + if (p.preferJoinType & preferBCJoin) > 0 { + errMsg = "The join can not push down to the MPP side, the broadcast_join() hint is invalid" + hasMppHints = true + } + if hasMppHints { + warning := ErrInternal.GenWithStack(errMsg) + p.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) + } } if prop.IsFlashProp() { return joins, true, nil @@ -2730,8 +2745,11 @@ func (la *LogicalAggregation) checkCanPushDownToMPP() bool { } } if hasUnsupportedDistinct { + warnErr := errors.New("Aggregation can not be pushed to storage layer in mpp mode because it contains agg function with distinct") if la.ctx.GetSessionVars().StmtCtx.InExplainStmt { - la.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("Aggregation can not be pushed to storage layer in mpp mode because it contains agg function with distinct")) + la.ctx.GetSessionVars().StmtCtx.AppendWarning(warnErr) + } else { + la.ctx.GetSessionVars().StmtCtx.AppendExtraWarning(warnErr) } return false } @@ -2855,6 +2873,21 @@ func (la *LogicalAggregation) getHashAggs(prop *property.PhysicalProperty) []Phy } if canPushDownToMPP { taskTypes = append(taskTypes, property.MppTaskType) + } else { + hasMppHints := false + var errMsg string + if la.aggHints.preferAggType&preferMPP1PhaseAgg > 0 { + errMsg = "The agg can not push down to the MPP side, the MPP_1PHASE_AGG() hint is invalid" + hasMppHints = true + } + if la.aggHints.preferAggType&preferMPP2PhaseAgg > 0 { + errMsg = "The agg can not push down to the MPP side, the MPP_2PHASE_AGG() hint is invalid" + hasMppHints = true + } + if hasMppHints { + warning := ErrInternal.GenWithStack(errMsg) + la.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) + } } if prop.IsFlashProp() { taskTypes = []property.TaskType{prop.TaskTp} diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index d0ca6e6f8e4cf..ddb905dc5c06b 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -55,7 +55,7 @@ func evalAstExpr(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error if val, ok := expr.(*driver.ValueExpr); ok { return val.Datum, nil } - newExpr, err := rewriteAstExpr(sctx, expr, nil, nil) + newExpr, err := rewriteAstExpr(sctx, expr, nil, nil, false) if err != nil { return types.Datum{}, err } @@ -63,13 +63,14 @@ func evalAstExpr(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error } // rewriteAstExpr rewrites ast expression directly. -func rewriteAstExpr(sctx sessionctx.Context, expr ast.ExprNode, schema *expression.Schema, names types.NameSlice) (expression.Expression, error) { +func rewriteAstExpr(sctx sessionctx.Context, expr ast.ExprNode, schema *expression.Schema, names types.NameSlice, allowCastArray bool) (expression.Expression, error) { var is infoschema.InfoSchema // in tests, it may be null if s, ok := sctx.GetInfoSchema().(infoschema.InfoSchema); ok { is = s } b, savedBlockNames := NewPlanBuilder().Init(sctx, is, &hint.BlockHintProcessor{}) + b.allowBuildCastArray = allowCastArray fakePlan := LogicalTableDual{}.Init(sctx, 0) if schema != nil { fakePlan.schema = schema @@ -1183,6 +1184,10 @@ func (er *expressionRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok er.disableFoldCounter-- } case *ast.FuncCastExpr: + if v.Tp.IsArray() && !er.b.allowBuildCastArray { + er.err = expression.ErrNotSupportedYet.GenWithStackByArgs("Use of CAST( .. AS .. ARRAY) outside of functional index in CREATE(non-SELECT)/ALTER TABLE or in general expressions") + return retNode, false + } arg := er.ctxStack[len(er.ctxStack)-1] er.err = expression.CheckArgsNotMultiColumnRow(arg) if er.err != nil { @@ -1195,7 +1200,11 @@ func (er *expressionRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok return retNode, false } - castFunction := expression.BuildCastFunction(er.sctx, arg, v.Tp) + castFunction, err := expression.BuildCastFunctionWithCheck(er.sctx, arg, v.Tp) + if err != nil { + er.err = err + return retNode, false + } if v.Tp.EvalType() == types.ETString { castFunction.SetCoercibility(expression.CoercibilityImplicit) if v.Tp.GetCharset() == charset.CharsetASCII { diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index afc5223b9be94..639bc15dbdc98 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -742,7 +742,7 @@ func (ds *DataSource) skylinePruning(prop *property.PhysicalProperty) []*candida } func (ds *DataSource) getPruningInfo(candidates []*candidatePath, prop *property.PhysicalProperty) string { - if !ds.ctx.GetSessionVars().StmtCtx.InVerboseExplain || len(candidates) == len(ds.possibleAccessPaths) { + if len(candidates) == len(ds.possibleAccessPaths) { return "" } if len(candidates) == 1 && len(candidates[0].path.Ranges) == 0 { @@ -889,10 +889,12 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter pruningInfo := ds.getPruningInfo(candidates, prop) defer func() { if err == nil && t != nil && !t.invalid() && pruningInfo != "" { - if ds.ctx.GetSessionVars().StmtCtx.OptimInfo == nil { - ds.ctx.GetSessionVars().StmtCtx.OptimInfo = make(map[int]string) + warnErr := errors.New(pruningInfo) + if ds.ctx.GetSessionVars().StmtCtx.InVerboseExplain { + ds.ctx.GetSessionVars().StmtCtx.AppendNote(warnErr) + } else { + ds.ctx.GetSessionVars().StmtCtx.AppendExtraNote(warnErr) } - ds.ctx.GetSessionVars().StmtCtx.OptimInfo[t.plan().ID()] = pruningInfo } }() diff --git a/planner/core/foreign_key.go b/planner/core/foreign_key.go index 00b87e11c01f8..f6cec0fa8f069 100644 --- a/planner/core/foreign_key.go +++ b/planner/core/foreign_key.go @@ -159,6 +159,17 @@ func (p *Insert) buildOnInsertFKTriggers(ctx sessionctx.Context, is infoschema.I if len(referredFKCascades) > 0 { fkCascades = append(fkCascades, referredFKCascades...) } + } else if p.IsReplace { + referredFKChecks, referredFKCascades, err := p.buildOnReplaceReferredFKTriggers(ctx, is, dbName, tblInfo) + if err != nil { + return err + } + if len(referredFKChecks) > 0 { + fkChecks = append(fkChecks, referredFKChecks...) + } + if len(referredFKCascades) > 0 { + fkCascades = append(fkCascades, referredFKCascades...) + } } for _, fk := range tblInfo.ForeignKeys { if fk.Version < 1 { @@ -186,6 +197,25 @@ func (p *Insert) buildOnDuplicateUpdateColumns() map[string]struct{} { return m } +func (p *Insert) buildOnReplaceReferredFKTriggers(ctx sessionctx.Context, is infoschema.InfoSchema, dbName string, tblInfo *model.TableInfo) ([]*FKCheck, []*FKCascade, error) { + referredFKs := is.GetTableReferredForeignKeys(dbName, tblInfo.Name.L) + fkChecks := make([]*FKCheck, 0, len(referredFKs)) + fkCascades := make([]*FKCascade, 0, len(referredFKs)) + for _, referredFK := range referredFKs { + fkCheck, fkCascade, err := buildOnDeleteOrUpdateFKTrigger(ctx, is, referredFK, FKCascadeOnDelete) + if err != nil { + return nil, nil, err + } + if fkCheck != nil { + fkChecks = append(fkChecks, fkCheck) + } + if fkCascade != nil { + fkCascades = append(fkCascades, fkCascade) + } + } + return fkChecks, fkCascades, nil +} + func (updt *Update) buildOnUpdateFKTriggers(ctx sessionctx.Context, is infoschema.InfoSchema, tblID2table map[int64]table.Table) error { if !ctx.GetSessionVars().ForeignKeyChecks { return nil @@ -414,7 +444,7 @@ func buildFKCheck(ctx sessionctx.Context, tbl table.Table, cols []model.CIStr, f } } - referTbIdxInfo := model.FindIndexByColumns(tblInfo, cols...) + referTbIdxInfo := model.FindIndexByColumns(tblInfo, tblInfo.Indices, cols...) if referTbIdxInfo == nil { return nil, failedErr } @@ -460,7 +490,7 @@ func buildFKCascade(ctx sessionctx.Context, tp FKCascadeType, referredFK *model. return fkCascade, nil } } - indexForFK := model.FindIndexByColumns(childTable.Meta(), fk.Cols...) + indexForFK := model.FindIndexByColumns(childTable.Meta(), childTable.Meta().Indices, fk.Cols...) if indexForFK == nil { return nil, errors.Errorf("Missing index for '%s' foreign key columns in the table '%s'", fk.Name, childTable.Meta().Name) } diff --git a/planner/core/fragment.go b/planner/core/fragment.go index c6aec17f21e6d..917f4392d9f9e 100644 --- a/planner/core/fragment.go +++ b/planner/core/fragment.go @@ -378,7 +378,7 @@ func (e *mppTaskGenerator) constructMPPTasksImpl(ctx context.Context, ts *Physic logutil.BgLogger().Warn("MPP store fail ttl is invalid", zap.Error(err)) ttl = 30 * time.Second } - metas, err := e.ctx.GetMPPClient().ConstructMPPTasks(ctx, req, e.ctx.GetSessionVars().MPPStoreLastFailTime, ttl) + metas, err := e.ctx.GetMPPClient().ConstructMPPTasks(ctx, req, ttl) if err != nil { return nil, errors.Trace(err) } diff --git a/planner/core/indexmerge_path.go b/planner/core/indexmerge_path.go new file mode 100644 index 0000000000000..51f950bf7d61a --- /dev/null +++ b/planner/core/indexmerge_path.go @@ -0,0 +1,469 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "fmt" + "math" + "strings" + + "github.com/pingcap/errors" + "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" + "github.com/pingcap/tidb/parser/mysql" + "github.com/pingcap/tidb/planner/util" + "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tidb/util/ranger" + "go.uber.org/zap" +) + +// generateIndexMergePath generates IndexMerge AccessPaths on this DataSource. +func (ds *DataSource) generateIndexMergePath() error { + // Consider the IndexMergePath. Now, we just generate `IndexMergePath` in DNF case. + // Use allConds instread of pushedDownConds, + // because we want to use IndexMerge even if some expr cannot be pushed to TiKV. + // We will create new Selection for exprs that cannot be pushed in convertToIndexMergeScan. + indexMergeConds := make([]expression.Expression, 0, len(ds.allConds)) + for _, expr := range ds.allConds { + indexMergeConds = append(indexMergeConds, expression.PushDownNot(ds.ctx, expr)) + } + + stmtCtx := ds.ctx.GetSessionVars().StmtCtx + isPossibleIdxMerge := len(indexMergeConds) > 0 && len(ds.possibleAccessPaths) > 1 + sessionAndStmtPermission := (ds.ctx.GetSessionVars().GetEnableIndexMerge() || len(ds.indexMergeHints) > 0) && !stmtCtx.NoIndexMergeHint + // We current do not consider `IndexMergePath`: + // 1. If there is an index path. + // 2. TODO: If there exists exprs that cannot be pushed down. This is to avoid wrongly estRow of Selection added by rule_predicate_push_down. + needConsiderIndexMerge := true + if len(ds.indexMergeHints) == 0 { + for i := 1; i < len(ds.possibleAccessPaths); i++ { + if len(ds.possibleAccessPaths[i].AccessConds) != 0 { + needConsiderIndexMerge = false + break + } + } + if needConsiderIndexMerge { + // PushDownExprs() will append extra warnings, which is annoying. So we reset warnings here. + warnings := stmtCtx.GetWarnings() + extraWarnings := stmtCtx.GetExtraWarnings() + _, remaining := expression.PushDownExprs(stmtCtx, indexMergeConds, ds.ctx.GetClient(), kv.UnSpecified) + stmtCtx.SetWarnings(warnings) + stmtCtx.SetExtraWarnings(extraWarnings) + if len(remaining) != 0 { + needConsiderIndexMerge = false + } + } + } + + if isPossibleIdxMerge && sessionAndStmtPermission && needConsiderIndexMerge && ds.tableInfo.TempTableType != model.TempTableLocal { + err := ds.generateAndPruneIndexMergePath(indexMergeConds, ds.indexMergeHints != nil) + if err != nil { + return err + } + } else if len(ds.indexMergeHints) > 0 { + ds.indexMergeHints = nil + var msg string + if !isPossibleIdxMerge { + msg = "No available filter or available index." + } else if !sessionAndStmtPermission { + msg = "Got no_index_merge hint or tidb_enable_index_merge is off." + } else if ds.tableInfo.TempTableType == model.TempTableLocal { + msg = "Cannot use IndexMerge on temporary table." + } + msg = fmt.Sprintf("IndexMerge is inapplicable or disabled. %s", msg) + stmtCtx.AppendWarning(errors.Errorf(msg)) + logutil.BgLogger().Debug(msg) + } + + return nil +} + +// getIndexMergeOrPath generates all possible IndexMergeOrPaths. +func (ds *DataSource) generateIndexMergeOrPaths(filters []expression.Expression) error { + usedIndexCount := len(ds.possibleAccessPaths) + for i, cond := range filters { + sf, ok := cond.(*expression.ScalarFunction) + if !ok || sf.FuncName.L != ast.LogicOr { + continue + } + var partialPaths = make([]*util.AccessPath, 0, usedIndexCount) + dnfItems := expression.FlattenDNFConditions(sf) + for _, item := range dnfItems { + cnfItems := expression.SplitCNFItems(item) + itemPaths := ds.accessPathsForConds(cnfItems, usedIndexCount) + if len(itemPaths) == 0 { + partialPaths = nil + break + } + partialPath, err := ds.buildIndexMergePartialPath(itemPaths) + if err != nil { + return err + } + if partialPath == nil { + partialPaths = nil + break + } + partialPaths = append(partialPaths, partialPath) + } + // If all of the partialPaths use the same index, we will not use the indexMerge. + singlePath := true + for i := len(partialPaths) - 1; i >= 1; i-- { + if partialPaths[i].Index != partialPaths[i-1].Index { + singlePath = false + break + } + } + if singlePath { + continue + } + if len(partialPaths) > 1 { + possiblePath := ds.buildIndexMergeOrPath(filters, partialPaths, i) + if possiblePath == nil { + return nil + } + + accessConds := make([]expression.Expression, 0, len(partialPaths)) + for _, p := range partialPaths { + indexCondsForP := p.AccessConds[:] + indexCondsForP = append(indexCondsForP, p.IndexFilters...) + if len(indexCondsForP) > 0 { + accessConds = append(accessConds, expression.ComposeCNFCondition(ds.ctx, indexCondsForP...)) + } + } + accessDNF := expression.ComposeDNFCondition(ds.ctx, accessConds...) + sel, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, []expression.Expression{accessDNF}, nil) + if err != nil { + logutil.BgLogger().Debug("something wrong happened, use the default selectivity", zap.Error(err)) + sel = SelectionFactor + } + possiblePath.CountAfterAccess = sel * ds.tableStats.RowCount + ds.possibleAccessPaths = append(ds.possibleAccessPaths, possiblePath) + } + } + return nil +} + +// isInIndexMergeHints returns true if the input index name is not excluded by the IndexMerge hints, which means either +// (1) there's no IndexMerge hint, (2) there's IndexMerge hint but no specified index names, or (3) the input index +// name is specified in the IndexMerge hints. +func (ds *DataSource) isInIndexMergeHints(name string) bool { + if len(ds.indexMergeHints) == 0 { + return true + } + for _, hint := range ds.indexMergeHints { + if hint.indexHint == nil || len(hint.indexHint.IndexNames) == 0 { + return true + } + for _, hintName := range hint.indexHint.IndexNames { + if strings.EqualFold(strings.ToLower(name), strings.ToLower(hintName.String())) { + return true + } + } + } + return false +} + +// indexMergeHintsHasSpecifiedIdx returns true if there's IndexMerge hint, and it has specified index names. +func (ds *DataSource) indexMergeHintsHasSpecifiedIdx() bool { + for _, hint := range ds.indexMergeHints { + if hint.indexHint == nil || len(hint.indexHint.IndexNames) == 0 { + continue + } + if len(hint.indexHint.IndexNames) > 0 { + return true + } + } + return false +} + +// indexMergeHintsHasSpecifiedIdx return true if the input index name is specified in the IndexMerge hint. +func (ds *DataSource) isSpecifiedInIndexMergeHints(name string) bool { + for _, hint := range ds.indexMergeHints { + if hint.indexHint == nil || len(hint.indexHint.IndexNames) == 0 { + continue + } + for _, hintName := range hint.indexHint.IndexNames { + if strings.EqualFold(strings.ToLower(name), strings.ToLower(hintName.String())) { + return true + } + } + } + return false +} + +// accessPathsForConds generates all possible index paths for conditions. +func (ds *DataSource) accessPathsForConds(conditions []expression.Expression, usedIndexCount int) []*util.AccessPath { + var results = make([]*util.AccessPath, 0, usedIndexCount) + for i := 0; i < usedIndexCount; i++ { + path := &util.AccessPath{} + if ds.possibleAccessPaths[i].IsTablePath() { + if !ds.isInIndexMergeHints("primary") { + continue + } + if ds.tableInfo.IsCommonHandle { + path.IsCommonHandlePath = true + path.Index = ds.possibleAccessPaths[i].Index + } else { + path.IsIntHandlePath = true + } + err := ds.deriveTablePathStats(path, conditions, true) + if err != nil { + logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err)) + continue + } + var unsignedIntHandle bool + if path.IsIntHandlePath && ds.tableInfo.PKIsHandle { + if pkColInfo := ds.tableInfo.GetPkColInfo(); pkColInfo != nil { + unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.GetFlag()) + } + } + // If the path contains a full range, ignore it. + if ranger.HasFullRange(path.Ranges, unsignedIntHandle) { + continue + } + // If we have point or empty range, just remove other possible paths. + if len(path.Ranges) == 0 || path.OnlyPointRange(ds.SCtx()) { + if len(results) == 0 { + results = append(results, path) + } else { + results[0] = path + results = results[:1] + } + break + } + } else { + path.Index = ds.possibleAccessPaths[i].Index + if !ds.isInIndexMergeHints(path.Index.Name.L) { + continue + } + err := ds.fillIndexPath(path, conditions) + if err != nil { + logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err)) + continue + } + ds.deriveIndexPathStats(path, conditions, true) + // If the path contains a full range, ignore it. + if ranger.HasFullRange(path.Ranges, false) { + continue + } + // If we have empty range, or point range on unique index, just remove other possible paths. + if len(path.Ranges) == 0 || (path.OnlyPointRange(ds.SCtx()) && path.Index.Unique) { + if len(results) == 0 { + results = append(results, path) + } else { + results[0] = path + results = results[:1] + } + break + } + } + results = append(results, path) + } + return results +} + +// buildIndexMergePartialPath chooses the best index path from all possible paths. +// Now we choose the index with minimal estimate row count. +func (ds *DataSource) buildIndexMergePartialPath(indexAccessPaths []*util.AccessPath) (*util.AccessPath, error) { + if len(indexAccessPaths) == 1 { + return indexAccessPaths[0], nil + } + + minEstRowIndex := 0 + minEstRow := math.MaxFloat64 + for i := 0; i < len(indexAccessPaths); i++ { + rc := indexAccessPaths[i].CountAfterAccess + if len(indexAccessPaths[i].IndexFilters) > 0 { + rc = indexAccessPaths[i].CountAfterIndex + } + if rc < minEstRow { + minEstRowIndex = i + minEstRow = rc + } + } + return indexAccessPaths[minEstRowIndex], nil +} + +// buildIndexMergeOrPath generates one possible IndexMergePath. +func (ds *DataSource) buildIndexMergeOrPath(filters []expression.Expression, partialPaths []*util.AccessPath, current int) *util.AccessPath { + indexMergePath := &util.AccessPath{PartialIndexPaths: partialPaths} + indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[:current]...) + indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[current+1:]...) + var addCurrentFilter bool + for _, path := range partialPaths { + // If any partial path contains table filters, we need to keep the whole DNF filter in the Selection. + if len(path.TableFilters) > 0 { + addCurrentFilter = true + } + // If any partial path's index filter cannot be pushed to TiKV, we should keep the whole DNF filter. + if len(path.IndexFilters) != 0 && !expression.CanExprsPushDown(ds.ctx.GetSessionVars().StmtCtx, path.IndexFilters, ds.ctx.GetClient(), kv.TiKV) { + addCurrentFilter = true + // Clear IndexFilter, the whole filter will be put in indexMergePath.TableFilters. + path.IndexFilters = nil + } + if len(path.TableFilters) != 0 && !expression.CanExprsPushDown(ds.ctx.GetSessionVars().StmtCtx, path.TableFilters, ds.ctx.GetClient(), kv.TiKV) { + addCurrentFilter = true + path.TableFilters = nil + } + } + if addCurrentFilter { + indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[current]) + } + return indexMergePath +} + +// generateIndexMergeAndPaths generates IndexMerge paths for `AND` (a.k.a. intersection type IndexMerge) +func (ds *DataSource) generateIndexMergeAndPaths(normalPathCnt int) *util.AccessPath { + // For now, we only consider intersection type IndexMerge when the index names are specified in the hints. + if !ds.indexMergeHintsHasSpecifiedIdx() { + return nil + } + + // 1. Collect partial paths from normal paths. + var partialPaths []*util.AccessPath + for i := 0; i < normalPathCnt; i++ { + originalPath := ds.possibleAccessPaths[i] + // No need to consider table path as a partial path. + if ds.possibleAccessPaths[i].IsTablePath() { + continue + } + if !ds.isSpecifiedInIndexMergeHints(originalPath.Index.Name.L) { + continue + } + // If the path contains a full range, ignore it. + if ranger.HasFullRange(originalPath.Ranges, false) { + continue + } + newPath := originalPath.Clone() + partialPaths = append(partialPaths, newPath) + } + if len(partialPaths) < 2 { + return nil + } + + // 2. Collect filters that can't be covered by the partial paths and deduplicate them. + finalFilters := make([]expression.Expression, 0) + partialFilters := make([]expression.Expression, 0, len(partialPaths)) + hashCodeSet := make(map[string]struct{}) + for _, path := range partialPaths { + // Classify filters into coveredConds and notCoveredConds. + coveredConds := make([]expression.Expression, 0, len(path.AccessConds)+len(path.IndexFilters)) + notCoveredConds := make([]expression.Expression, 0, len(path.IndexFilters)+len(path.TableFilters)) + // AccessConds can be covered by partial path. + coveredConds = append(coveredConds, path.AccessConds...) + for i, cond := range path.IndexFilters { + // IndexFilters can be covered by partial path if it can be pushed down to TiKV. + if !expression.CanExprsPushDown(ds.ctx.GetSessionVars().StmtCtx, []expression.Expression{cond}, ds.ctx.GetClient(), kv.TiKV) { + path.IndexFilters = append(path.IndexFilters[:i], path.IndexFilters[i+1:]...) + notCoveredConds = append(notCoveredConds, cond) + } else { + coveredConds = append(coveredConds, cond) + } + } + // TableFilters can't be covered by partial path. + notCoveredConds = append(notCoveredConds, path.TableFilters...) + + // Record covered filters in hashCodeSet. + // Note that we only record filters that not appear in the notCoveredConds. It's possible that a filter appear + // in both coveredConds and notCoveredConds (e.g. because of prefix index). So we need this extra check to + // avoid wrong deduplication. + notCoveredHashCodeSet := make(map[string]struct{}) + for _, cond := range notCoveredConds { + hashCode := string(cond.HashCode(ds.ctx.GetSessionVars().StmtCtx)) + notCoveredHashCodeSet[hashCode] = struct{}{} + } + for _, cond := range coveredConds { + hashCode := string(cond.HashCode(ds.ctx.GetSessionVars().StmtCtx)) + if _, ok := notCoveredHashCodeSet[hashCode]; !ok { + hashCodeSet[hashCode] = struct{}{} + } + } + + finalFilters = append(finalFilters, notCoveredConds...) + partialFilters = append(partialFilters, coveredConds...) + } + + // Remove covered filters from finalFilters and deduplicate finalFilters. + dedupedFinalFilters := make([]expression.Expression, 0, len(finalFilters)) + for _, cond := range finalFilters { + hashCode := string(cond.HashCode(ds.ctx.GetSessionVars().StmtCtx)) + if _, ok := hashCodeSet[hashCode]; !ok { + dedupedFinalFilters = append(dedupedFinalFilters, cond) + hashCodeSet[hashCode] = struct{}{} + } + } + + // 3. Estimate the row count after partial paths. + sel, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, partialFilters, nil) + if err != nil { + logutil.BgLogger().Debug("something wrong happened, use the default selectivity", zap.Error(err)) + sel = SelectionFactor + } + + indexMergePath := &util.AccessPath{ + PartialIndexPaths: partialPaths, + IndexMergeIsIntersection: true, + TableFilters: dedupedFinalFilters, + CountAfterAccess: sel * ds.tableStats.RowCount, + } + return indexMergePath +} + +func (ds *DataSource) generateAndPruneIndexMergePath(indexMergeConds []expression.Expression, needPrune bool) error { + regularPathCount := len(ds.possibleAccessPaths) + // 1. Generate possible IndexMerge paths for `OR`. + err := ds.generateIndexMergeOrPaths(indexMergeConds) + if err != nil { + return err + } + // 2. Generate possible IndexMerge paths for `AND`. + indexMergeAndPath := ds.generateIndexMergeAndPaths(regularPathCount) + if indexMergeAndPath != nil { + ds.possibleAccessPaths = append(ds.possibleAccessPaths, indexMergeAndPath) + } + + // 3. If needed, append a warning if no IndexMerge is generated. + + // If without hints, it means that `enableIndexMerge` is true + if len(ds.indexMergeHints) == 0 { + return nil + } + // With hints and without generated IndexMerge paths + if regularPathCount == len(ds.possibleAccessPaths) { + ds.indexMergeHints = nil + ds.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("IndexMerge is inapplicable")) + return nil + } + + // 4. If needPrune is true, prune non-IndexMerge paths. + + // Do not need to consider the regular paths in find_best_task(). + // So we can use index merge's row count as DataSource's row count. + if needPrune { + ds.possibleAccessPaths = ds.possibleAccessPaths[regularPathCount:] + minRowCount := ds.possibleAccessPaths[0].CountAfterAccess + for _, path := range ds.possibleAccessPaths { + if minRowCount < path.CountAfterAccess { + minRowCount = path.CountAfterAccess + } + } + if ds.stats.RowCount > minRowCount { + ds.stats = ds.tableStats.ScaleByExpectCnt(minRowCount) + } + } + return nil +} diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 7e5b13b588e16..640c0f04630c1 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -813,9 +813,11 @@ func TestMPPHints(t *testing.T) { tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("set tidb_cost_model_version=2") - tk.MustExec("create table t (a int, b int, c int)") + tk.MustExec("create table t (a int, b int, c int, index idx_a(a), index idx_b(b))") tk.MustExec("alter table t set tiflash replica 1") tk.MustExec("set @@session.tidb_allow_mpp=ON") + tk.MustExec("create definer='root'@'localhost' view v as select a, sum(b) from t group by a, c;") + tk.MustExec("create definer='root'@'localhost' view v1 as select t1.a from t t1, t t2 where t1.a=t2.a;") tb := external.GetTableByName(t, tk, "test", "t") err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true) require.NoError(t, err) @@ -824,6 +826,7 @@ func TestMPPHints(t *testing.T) { var output []struct { SQL string Plan []string + Warn []string } planSuiteData := core.GetPlanSuiteData() @@ -833,11 +836,101 @@ func TestMPPHints(t *testing.T) { testdata.OnRecord(func() { output[i].SQL = ts output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery("explain format = 'brief' " + ts).Rows()) + output[i].Warn = testdata.ConvertSQLWarnToStrings(tk.Session().GetSessionVars().StmtCtx.GetWarnings()) }) tk.MustQuery("explain format = 'brief' " + ts).Check(testkit.Rows(output[i].Plan...)) + require.Equal(t, output[i].Warn, testdata.ConvertSQLWarnToStrings(tk.Session().GetSessionVars().StmtCtx.GetWarnings())) } } +func TestMPPHintsScope(t *testing.T) { + store := testkit.CreateMockStore(t, withMockTiFlash(2)) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("set tidb_cost_model_version=2") + tk.MustExec("create table t (a int, b int, c int, index idx_a(a), index idx_b(b))") + tk.MustExec("select /*+ MPP_1PHASE_AGG() */ a, sum(b) from t group by a, c") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1815 The agg can not push down to the MPP side, the MPP_1PHASE_AGG() hint is invalid")) + tk.MustExec("select /*+ MPP_2PHASE_AGG() */ a, sum(b) from t group by a, c") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1815 The agg can not push down to the MPP side, the MPP_2PHASE_AGG() hint is invalid")) + tk.MustExec("select /*+ shuffle_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1815 The join can not push down to the MPP side, the shuffle_join() hint is invalid")) + tk.MustExec("select /*+ broadcast_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1815 The join can not push down to the MPP side, the broadcast_join() hint is invalid")) + tk.MustExec("alter table t set tiflash replica 1") + tb := external.GetTableByName(t, tk, "test", "t") + err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true) + require.NoError(t, err) + tk.MustExec("set @@session.tidb_allow_mpp=true") + tk.MustExec("explain select /*+ MPP_1PHASE_AGG() */ a, sum(b) from t group by a, c") + tk.MustQuery("show warnings").Check(testkit.Rows()) + tk.MustExec("explain select /*+ MPP_2PHASE_AGG() */ a, sum(b) from t group by a, c") + tk.MustQuery("show warnings").Check(testkit.Rows()) + tk.MustExec("explain select /*+ shuffle_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a") + tk.MustQuery("show warnings").Check(testkit.Rows()) + tk.MustExec("explain select /*+ broadcast_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a") + tk.MustQuery("show warnings").Check(testkit.Rows()) + + tk.MustExec("set @@session.tidb_allow_mpp=false") + tk.MustExec("explain select /*+ MPP_1PHASE_AGG() */ a, sum(b) from t group by a, c") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1815 The agg can not push down to the MPP side, the MPP_1PHASE_AGG() hint is invalid")) + tk.MustExec("explain select /*+ MPP_2PHASE_AGG() */ a, sum(b) from t group by a, c") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1815 The agg can not push down to the MPP side, the MPP_2PHASE_AGG() hint is invalid")) + tk.MustExec("explain select /*+ shuffle_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1815 The join can not push down to the MPP side, the shuffle_join() hint is invalid")) + tk.MustExec("explain select /*+ broadcast_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1815 The join can not push down to the MPP side, the broadcast_join() hint is invalid")) +} + +func TestMPPHintsWithBinding(t *testing.T) { + store := testkit.CreateMockStore(t, withMockTiFlash(2)) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("set tidb_cost_model_version=2") + tk.MustExec("create table t (a int, b int, c int)") + tk.MustExec("alter table t set tiflash replica 1") + tk.MustExec("set @@session.tidb_allow_mpp=ON") + tb := external.GetTableByName(t, tk, "test", "t") + err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true) + require.NoError(t, err) + + tk.MustExec("explain select a, sum(b) from t group by a, c") + tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("0")) + tk.MustExec("create global binding for select a, sum(b) from t group by a, c using select /*+ read_from_storage(tiflash[t]), MPP_1PHASE_AGG() */ a, sum(b) from t group by a, c;") + tk.MustExec("explain select a, sum(b) from t group by a, c") + tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1")) + res := tk.MustQuery("show global bindings").Rows() + require.Equal(t, res[0][0], "select `a` , sum ( `b` ) from `test` . `t` group by `a` , `c`") + require.Equal(t, res[0][1], "SELECT /*+ read_from_storage(tiflash[`t`]) MPP_1PHASE_AGG()*/ `a`,sum(`b`) FROM `test`.`t` GROUP BY `a`,`c`") + tk.MustExec("create global binding for select a, sum(b) from t group by a, c using select /*+ read_from_storage(tiflash[t]), MPP_2PHASE_AGG() */ a, sum(b) from t group by a, c;") + tk.MustExec("explain select a, sum(b) from t group by a, c") + tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1")) + res = tk.MustQuery("show global bindings").Rows() + require.Equal(t, res[0][0], "select `a` , sum ( `b` ) from `test` . `t` group by `a` , `c`") + require.Equal(t, res[0][1], "SELECT /*+ read_from_storage(tiflash[`t`]) MPP_2PHASE_AGG()*/ `a`,sum(`b`) FROM `test`.`t` GROUP BY `a`,`c`") + tk.MustExec("drop global binding for select a, sum(b) from t group by a, c;") + res = tk.MustQuery("show global bindings").Rows() + require.Equal(t, len(res), 0) + + tk.MustExec("explain select * from t t1, t t2 where t1.a=t2.a") + tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("0")) + tk.MustExec("create global binding for select * from t t1, t t2 where t1.a=t2.a using select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a") + tk.MustExec("explain select * from t t1, t t2 where t1.a=t2.a") + tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1")) + res = tk.MustQuery("show global bindings").Rows() + require.Equal(t, res[0][0], "select * from ( `test` . `t` as `t1` ) join `test` . `t` as `t2` where `t1` . `a` = `t2` . `a`") + require.Equal(t, res[0][1], "SELECT /*+ read_from_storage(tiflash[`t1`, `t2`]) shuffle_join(`t1`, `t2`)*/ * FROM (`test`.`t` AS `t1`) JOIN `test`.`t` AS `t2` WHERE `t1`.`a` = `t2`.`a`") + tk.MustExec("create global binding for select * from t t1, t t2 where t1.a=t2.a using select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a;") + tk.MustExec("explain select * from t t1, t t2 where t1.a=t2.a") + tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1")) + res = tk.MustQuery("show global bindings").Rows() + require.Equal(t, res[0][0], "select * from ( `test` . `t` as `t1` ) join `test` . `t` as `t2` where `t1` . `a` = `t2` . `a`") + require.Equal(t, res[0][1], "SELECT /*+ read_from_storage(tiflash[`t1`, `t2`]) broadcast_join(`t1`, `t2`)*/ * FROM (`test`.`t` AS `t1`) JOIN `test`.`t` AS `t2` WHERE `t1`.`a` = `t2`.`a`") + tk.MustExec("drop global binding for select * from t t1, t t2 where t1.a=t2.a;") + res = tk.MustQuery("show global bindings").Rows() + require.Equal(t, len(res), 0) +} + func TestHintScope(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/planner/core/plan_cache.go b/planner/core/plan_cache.go index d9df658aaa46e..ab4eb4e4912ab 100644 --- a/planner/core/plan_cache.go +++ b/planner/core/plan_cache.go @@ -115,6 +115,10 @@ func planCachePreprocess(ctx context.Context, sctx sessionctx.Context, isNonPrep func GetPlanFromSessionPlanCache(ctx context.Context, sctx sessionctx.Context, isNonPrepared bool, is infoschema.InfoSchema, stmt *PlanCacheStmt, params []expression.Expression) (plan Plan, names []*types.FieldName, err error) { + if v := ctx.Value("____GetPlanFromSessionPlanCacheErr"); v != nil { // for testing + return nil, nil, errors.New("____GetPlanFromSessionPlanCacheErr") + } + if err := planCachePreprocess(ctx, sctx, isNonPrepared, is, stmt, params); err != nil { return nil, nil, err } diff --git a/planner/core/plan_cache_param.go b/planner/core/plan_cache_param.go index 7c79b2a6416a0..9094edec621c0 100644 --- a/planner/core/plan_cache_param.go +++ b/planner/core/plan_cache_param.go @@ -15,6 +15,7 @@ package core import ( + "context" "errors" "strings" "sync" @@ -70,7 +71,7 @@ func (pr *paramReplacer) Reset() { pr.params = nil } // ParameterizeAST parameterizes this StmtNode. // e.g. `select * from t where a<10 and b<23` --> `select * from t where a `select * from t where a<10 and b<23`. -func RestoreASTWithParams(_ sessionctx.Context, stmt ast.StmtNode, params []*driver.ValueExpr) error { +func RestoreASTWithParams(ctx context.Context, _ sessionctx.Context, stmt ast.StmtNode, params []*driver.ValueExpr) error { + if v := ctx.Value("____RestoreASTWithParamsErr"); v != nil { + return errors.New("____RestoreASTWithParamsErr") + } + pr := paramRestorerPool.Get().(*paramRestorer) defer func() { pr.Reset() diff --git a/planner/core/plan_cache_param_test.go b/planner/core/plan_cache_param_test.go index 5c65b89767a60..ee4a8e9ae65c5 100644 --- a/planner/core/plan_cache_param_test.go +++ b/planner/core/plan_cache_param_test.go @@ -15,6 +15,7 @@ package core import ( + "context" "strings" "testing" @@ -61,7 +62,7 @@ func TestParameterize(t *testing.T) { for _, c := range cases { stmt, err := parser.New().ParseOneStmt(c.sql, "", "") require.Nil(t, err) - paramSQL, params, err := ParameterizeAST(sctx, stmt) + paramSQL, params, err := ParameterizeAST(context.Background(), sctx, stmt) require.Nil(t, err) require.Equal(t, c.paramSQL, paramSQL) require.Equal(t, len(c.params), len(params)) @@ -69,7 +70,7 @@ func TestParameterize(t *testing.T) { require.Equal(t, c.params[i], params[i].Datum.GetValue()) } - err = RestoreASTWithParams(sctx, stmt, params) + err = RestoreASTWithParams(context.Background(), sctx, stmt, params) require.Nil(t, err) var buf strings.Builder rCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, &buf) diff --git a/planner/core/plan_cache_test.go b/planner/core/plan_cache_test.go index f541c441fd4f5..a480c583d0434 100644 --- a/planner/core/plan_cache_test.go +++ b/planner/core/plan_cache_test.go @@ -15,6 +15,7 @@ package core_test import ( + "context" "errors" "fmt" "math/rand" @@ -81,6 +82,76 @@ func TestInitLRUWithSystemVar(t *testing.T) { require.NotNil(t, lru) } +func TestNonPreparedPlanCacheWithExplain(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec(`use test`) + tk.MustExec("create table t(a int)") + tk.MustExec("set tidb_enable_non_prepared_plan_cache=1") + tk.MustExec("select * from t where a=1") // cache this plan + + tk.MustQuery("explain select * from t where a=2").Check(testkit.Rows( + `Selection_8 10.00 root eq(test.t.a, 2)`, + `└─TableReader_7 10.00 root data:Selection_6`, + ` └─Selection_6 10.00 cop[tikv] eq(test.t.a, 2)`, + ` └─TableFullScan_5 10000.00 cop[tikv] table:t keep order:false, stats:pseudo`)) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) + + tk.MustQuery("explain format=verbose select * from t where a=2").Check(testkit.Rows( + `Selection_8 10.00 169474.57 root eq(test.t.a, 2)`, + `└─TableReader_7 10.00 168975.57 root data:Selection_6`, + ` └─Selection_6 10.00 2534000.00 cop[tikv] eq(test.t.a, 2)`, + ` └─TableFullScan_5 10000.00 2035000.00 cop[tikv] table:t keep order:false, stats:pseudo`)) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) + + tk.MustQuery("explain analyze select * from t where a=2").CheckAt([]int{0, 1, 2, 3}, [][]interface{}{ + {"Selection_8", "10.00", "0", "root"}, + {"└─TableReader_7", "10.00", "0", "root"}, + {" └─Selection_6", "10.00", "0", "cop[tikv]"}, + {" └─TableFullScan_5", "10000.00", "0", "cop[tikv]"}, + }) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) +} + +func TestNonPreparedPlanCacheFallback(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec(`use test`) + tk.MustExec(`create table t (a int)`) + for i := 0; i < 5; i++ { + tk.MustExec(fmt.Sprintf("insert into t values (%v)", i)) + } + tk.MustExec("set tidb_enable_non_prepared_plan_cache=1") + + // inject a fault to GeneratePlanCacheStmtWithAST + ctx := context.WithValue(context.Background(), "____GeneratePlanCacheStmtWithASTErr", struct{}{}) + tk.MustQueryWithContext(ctx, "select * from t where a in (1, 2)").Sort().Check(testkit.Rows("1", "2")) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) // cannot generate PlanCacheStmt + tk.MustQueryWithContext(ctx, "select * from t where a in (1, 3)").Sort().Check(testkit.Rows("1", "3")) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) // cannot generate PlanCacheStmt + tk.MustQuery("select * from t where a in (1, 2)").Sort().Check(testkit.Rows("1", "2")) + tk.MustQuery("select * from t where a in (1, 3)").Sort().Check(testkit.Rows("1", "3")) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) // no error + + // inject a fault to GetPlanFromSessionPlanCache + tk.MustQuery("select * from t where a=1").Check(testkit.Rows("1")) // cache this plan + tk.MustQuery("select * from t where a=2").Check(testkit.Rows("2")) // plan from cache + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) + ctx = context.WithValue(context.Background(), "____GetPlanFromSessionPlanCacheErr", struct{}{}) + tk.MustQueryWithContext(ctx, "select * from t where a=3").Check(testkit.Rows("3")) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) // fallback to the normal opt-path + tk.MustQueryWithContext(ctx, "select * from t where a=4").Check(testkit.Rows("4")) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) // fallback to the normal opt-path + tk.MustQueryWithContext(context.Background(), "select * from t where a=0").Check(testkit.Rows("0")) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) // use the cached plan if no error + + // inject a fault to RestoreASTWithParams + ctx = context.WithValue(context.Background(), "____GetPlanFromSessionPlanCacheErr", struct{}{}) + ctx = context.WithValue(ctx, "____RestoreASTWithParamsErr", struct{}{}) + _, err := tk.ExecWithContext(ctx, "select * from t where a=1") + require.NotNil(t, err) +} + func TestNonPreparedPlanCacheBasically(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -150,6 +221,29 @@ func TestIssue38533(t *testing.T) { tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) } +func TestIgnoreInsertStmt(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t (a int)") + + // do not cache native insert-stmt + tk.MustExec("prepare st from 'insert into t values (1)'") + tk.MustExec("execute st") + tk.MustExec("execute st") + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) + + // ignore-hint in insert-stmt can work + tk.MustExec("prepare st from 'insert into t select * from t'") + tk.MustExec("execute st") + tk.MustExec("execute st") + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) + tk.MustExec("prepare st from 'insert /*+ ignore_plan_cache() */ into t select * from t'") + tk.MustExec("execute st") + tk.MustExec("execute st") + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) +} + func TestIssue38710(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/planner/core/plan_cache_utils.go b/planner/core/plan_cache_utils.go index 2b1621857b9ca..8dc867316207d 100644 --- a/planner/core/plan_cache_utils.go +++ b/planner/core/plan_cache_utils.go @@ -64,17 +64,23 @@ func (e *paramMarkerExtractor) Leave(in ast.Node) (ast.Node, bool) { } // GeneratePlanCacheStmtWithAST generates the PlanCacheStmt structure for this AST. -func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, stmt ast.StmtNode) (*PlanCacheStmt, Plan, int, error) { +// paramSQL is the corresponding parameterized sql like 'select * from t where a?'. +// paramStmt is the Node of paramSQL. +func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, paramSQL string, paramStmt ast.StmtNode) (*PlanCacheStmt, Plan, int, error) { + if v := ctx.Value("____GeneratePlanCacheStmtWithASTErr"); v != nil { // for testing + return nil, nil, 0, errors.New("____GeneratePlanCacheStmtWithASTErr") + } + vars := sctx.GetSessionVars() var extractor paramMarkerExtractor - stmt.Accept(&extractor) + paramStmt.Accept(&extractor) // DDL Statements can not accept parameters - if _, ok := stmt.(ast.DDLNode); ok && len(extractor.markers) > 0 { + if _, ok := paramStmt.(ast.DDLNode); ok && len(extractor.markers) > 0 { return nil, nil, 0, ErrPrepareDDL } - switch stmt.(type) { + switch paramStmt.(type) { case *ast.LoadDataStmt, *ast.PrepareStmt, *ast.ExecuteStmt, *ast.DeallocateStmt, *ast.NonTransactionalDMLStmt: return nil, nil, 0, ErrUnsupportedPs } @@ -86,7 +92,7 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, } ret := &PreprocessorReturn{} - err := Preprocess(ctx, sctx, stmt, InPrepare, WithPreprocessorReturn(ret)) + err := Preprocess(ctx, sctx, paramStmt, InPrepare, WithPreprocessorReturn(ret)) if err != nil { return nil, nil, 0, err } @@ -103,8 +109,8 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, } prepared := &ast.Prepared{ - Stmt: stmt, - StmtType: ast.GetStmtLabel(stmt), + Stmt: paramStmt, + StmtType: ast.GetStmtLabel(paramStmt), Params: extractor.markers, SchemaVersion: ret.InfoSchema.SchemaMetaVersion(), } @@ -117,12 +123,12 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, if !vars.EnablePreparedPlanCache { prepared.UseCache = false } else { - cacheable, reason := CacheableWithCtx(sctx, stmt, ret.InfoSchema) + cacheable, reason := CacheableWithCtx(sctx, paramStmt, ret.InfoSchema) prepared.UseCache = cacheable if !cacheable { sctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("skip plan-cache: " + reason)) } - selectStmtNode, normalizedSQL4PC, digest4PC, err = ExtractSelectAndNormalizeDigest(stmt, vars.CurrentDB) + selectStmtNode, normalizedSQL4PC, digest4PC, err = ExtractSelectAndNormalizeDigest(paramStmt, vars.CurrentDB) if err != nil || selectStmtNode == nil { normalizedSQL4PC = "" digest4PC = "" @@ -138,7 +144,7 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, var p Plan destBuilder, _ := NewPlanBuilder().Init(sctx, ret.InfoSchema, &hint.BlockHintProcessor{}) - p, err = destBuilder.Build(ctx, stmt) + p, err = destBuilder.Build(ctx, paramStmt) if err != nil { return nil, nil, 0, err } @@ -146,7 +152,7 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, preparedObj := &PlanCacheStmt{ PreparedAst: prepared, StmtDB: vars.CurrentDB, - StmtText: stmt.Text(), + StmtText: paramSQL, VisitInfos: destBuilder.GetVisitInfo(), NormalizedSQL: normalizedSQL, SQLDigest: digest, diff --git a/planner/core/plan_cacheable_checker.go b/planner/core/plan_cacheable_checker.go index 3412ace379271..041509d224792 100644 --- a/planner/core/plan_cacheable_checker.go +++ b/planner/core/plan_cacheable_checker.go @@ -94,6 +94,21 @@ func (checker *cacheableChecker) Enter(in ast.Node) (out ast.Node, skipChildren return in, true } } + case *ast.InsertStmt: + if node.Select == nil { + // do not cache insert-values-stmt like 'insert into t values (...)' since + // no performance benefit and to save memory. + checker.cacheable = false + checker.reason = "ignore insert-values-stmt" + return in, true + } + for _, hints := range node.TableHints { + if hints.HintName.L == HintIgnorePlanCache { + checker.cacheable = false + checker.reason = "ignore plan cache by hint" + return in, true + } + } case *ast.VariableExpr, *ast.ExistsSubqueryExpr, *ast.SubqueryExpr: checker.cacheable = false checker.reason = "query has sub-queries is un-cacheable" diff --git a/planner/core/plan_cacheable_checker_test.go b/planner/core/plan_cacheable_checker_test.go index 658537bde4188..e87a08592eb16 100644 --- a/planner/core/plan_cacheable_checker_test.go +++ b/planner/core/plan_cacheable_checker_test.go @@ -53,7 +53,9 @@ func TestCacheable(t *testing.T) { tableRefsClause := &ast.TableRefsClause{TableRefs: &ast.Join{Left: &ast.TableSource{Source: tbl}}} // test InsertStmt - stmt = &ast.InsertStmt{Table: tableRefsClause} + stmt = &ast.InsertStmt{Table: tableRefsClause} // insert-values-stmt + require.False(t, core.Cacheable(stmt, is)) + stmt = &ast.InsertStmt{Table: tableRefsClause, Select: &ast.SelectStmt{}} // insert-select-stmt require.True(t, core.Cacheable(stmt, is)) // test DeleteStmt diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 9201f953bdcdc..238e2a687d758 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -577,6 +577,9 @@ type PlanBuilder struct { // disableSubQueryPreprocessing indicates whether to pre-process uncorrelated sub-queries in rewriting stage. disableSubQueryPreprocessing bool + + // allowBuildCastArray indicates whether allow cast(... as ... array). + allowBuildCastArray bool } type handleColHelper struct { @@ -697,6 +700,14 @@ func (p PlanBuilderOptNoExecution) Apply(builder *PlanBuilder) { builder.disableSubQueryPreprocessing = true } +// PlanBuilderOptAllowCastArray means the plan builder should allow build cast(... as ... array). +type PlanBuilderOptAllowCastArray struct{} + +// Apply implements the interface PlanBuilderOpt. +func (p PlanBuilderOptAllowCastArray) Apply(builder *PlanBuilder) { + builder.allowBuildCastArray = true +} + // NewPlanBuilder creates a new PlanBuilder. func NewPlanBuilder(opts ...PlanBuilderOpt) *PlanBuilder { builder := &PlanBuilder{ @@ -1003,10 +1014,20 @@ func (b *PlanBuilder) buildDropBindPlan(v *ast.DropBindingStmt) (Plan, error) { } func (b *PlanBuilder) buildSetBindingStatusPlan(v *ast.SetBindingStmt) (Plan, error) { - p := &SQLBindPlan{ - SQLBindOp: OpSetBindingStatus, - NormdOrigSQL: parser.Normalize(utilparser.RestoreWithDefaultDB(v.OriginNode, b.ctx.GetSessionVars().CurrentDB, v.OriginNode.Text())), - Db: utilparser.GetDefaultDB(v.OriginNode, b.ctx.GetSessionVars().CurrentDB), + var p *SQLBindPlan + if v.OriginNode != nil { + p = &SQLBindPlan{ + SQLBindOp: OpSetBindingStatus, + NormdOrigSQL: parser.Normalize(utilparser.RestoreWithDefaultDB(v.OriginNode, b.ctx.GetSessionVars().CurrentDB, v.OriginNode.Text())), + Db: utilparser.GetDefaultDB(v.OriginNode, b.ctx.GetSessionVars().CurrentDB), + } + } else if v.SQLDigest != "" { + p = &SQLBindPlan{ + SQLBindOp: OpSetBindingStatusByDigest, + SQLDigest: v.SQLDigest, + } + } else { + return nil, errors.New("sql digest is empty") } switch v.BindingStatusType { case ast.BindingStatusTypeEnabled: @@ -4511,6 +4532,14 @@ func (b *PlanBuilder) buildDDL(ctx context.Context, node ast.DDLNode) (Plan, err } b.visitInfo = appendVisitInfo(b.visitInfo, mysql.UpdatePriv, mysql.SystemDB, "stats_extended", "", authErr) + } else if spec.Tp == ast.AlterTableAddConstraint { + if b.ctx.GetSessionVars().User != nil && spec.Constraint != nil && + spec.Constraint.Tp == ast.ConstraintForeignKey && spec.Constraint.Refer != nil { + authErr = ErrTableaccessDenied.GenWithStackByArgs("REFERENCES", b.ctx.GetSessionVars().User.AuthUsername, + b.ctx.GetSessionVars().User.AuthHostname, spec.Constraint.Refer.Table.Name.L) + b.visitInfo = appendVisitInfo(b.visitInfo, mysql.ReferencesPriv, spec.Constraint.Refer.Table.Schema.L, + spec.Constraint.Refer.Table.Name.L, "", authErr) + } } } case *ast.AlterSequenceStmt: diff --git a/planner/core/stats.go b/planner/core/stats.go index f377feac91030..96065fccf12b5 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -22,9 +22,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/expression" - "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/parser/ast" - "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/planner/util" @@ -343,35 +340,37 @@ func (ds *DataSource) derivePathStatsAndTryHeuristics() error { if selected != nil { ds.possibleAccessPaths[0] = selected ds.possibleAccessPaths = ds.possibleAccessPaths[:1] - if ds.ctx.GetSessionVars().StmtCtx.InVerboseExplain { - var tableName string - if ds.TableAsName.O == "" { - tableName = ds.tableInfo.Name.O + var tableName string + if ds.TableAsName.O == "" { + tableName = ds.tableInfo.Name.O + } else { + tableName = ds.TableAsName.O + } + var sb strings.Builder + if selected.IsTablePath() { + // TODO: primary key / handle / real name? + sb.WriteString(fmt.Sprintf("handle of %s is selected since the path only has point ranges", tableName)) + } else { + if selected.Index.Unique { + sb.WriteString("unique ") + } + sb.WriteString(fmt.Sprintf("index %s of %s is selected since the path", selected.Index.Name.O, tableName)) + if isRefinedPath { + sb.WriteString(" only fetches limited number of rows") } else { - tableName = ds.TableAsName.O + sb.WriteString(" only has point ranges") } - if selected.IsTablePath() { - // TODO: primary key / handle / real name? - ds.ctx.GetSessionVars().StmtCtx.AppendNote(fmt.Errorf("handle of %s is selected since the path only has point ranges", tableName)) + if selected.IsSingleScan { + sb.WriteString(" with single scan") } else { - var sb strings.Builder - if selected.Index.Unique { - sb.WriteString("unique ") - } - sb.WriteString(fmt.Sprintf("index %s of %s is selected since the path", selected.Index.Name.O, tableName)) - if isRefinedPath { - sb.WriteString(" only fetches limited number of rows") - } else { - sb.WriteString(" only has point ranges") - } - if selected.IsSingleScan { - sb.WriteString(" with single scan") - } else { - sb.WriteString(" with double scan") - } - ds.ctx.GetSessionVars().StmtCtx.AppendNote(errors.New(sb.String())) + sb.WriteString(" with double scan") } } + if ds.ctx.GetSessionVars().StmtCtx.InVerboseExplain { + ds.ctx.GetSessionVars().StmtCtx.AppendNote(errors.New(sb.String())) + } else { + ds.ctx.GetSessionVars().StmtCtx.AppendExtraNote(errors.New(sb.String())) + } } return nil } @@ -409,107 +408,13 @@ func (ds *DataSource) DeriveStats(_ []*property.StatsInfo, _ *expression.Schema, return nil, err } - // Consider the IndexMergePath. Now, we just generate `IndexMergePath` in DNF case. - // Use allConds instread of pushedDownConds, - // because we want to use IndexMerge even if some expr cannot be pushed to TiKV. - // We will create new Selection for exprs that cannot be pushed in convertToIndexMergeScan. - indexMergeConds := make([]expression.Expression, 0, len(ds.allConds)) - for _, expr := range ds.allConds { - indexMergeConds = append(indexMergeConds, expression.PushDownNot(ds.ctx, expr)) - } - - stmtCtx := ds.ctx.GetSessionVars().StmtCtx - isPossibleIdxMerge := len(indexMergeConds) > 0 && len(ds.possibleAccessPaths) > 1 - sessionAndStmtPermission := (ds.ctx.GetSessionVars().GetEnableIndexMerge() || len(ds.indexMergeHints) > 0) && !stmtCtx.NoIndexMergeHint - // We current do not consider `IndexMergePath`: - // 1. If there is an index path. - // 2. TODO: If there exists exprs that cannot be pushed down. This is to avoid wrongly estRow of Selection added by rule_predicate_push_down. - needConsiderIndexMerge := true - if len(ds.indexMergeHints) == 0 { - for i := 1; i < len(ds.possibleAccessPaths); i++ { - if len(ds.possibleAccessPaths[i].AccessConds) != 0 { - needConsiderIndexMerge = false - break - } - } - if needConsiderIndexMerge { - // PushDownExprs() will append extra warnings, which is annoying. So we reset warnings here. - warnings := stmtCtx.GetWarnings() - _, remaining := expression.PushDownExprs(stmtCtx, indexMergeConds, ds.ctx.GetClient(), kv.UnSpecified) - stmtCtx.SetWarnings(warnings) - if len(remaining) != 0 { - needConsiderIndexMerge = false - } - } + if err := ds.generateIndexMergePath(); err != nil { + return nil, err } - if isPossibleIdxMerge && sessionAndStmtPermission && needConsiderIndexMerge && ds.tableInfo.TempTableType != model.TempTableLocal { - err := ds.generateAndPruneIndexMergePath(indexMergeConds, ds.indexMergeHints != nil) - if err != nil { - return nil, err - } - } else if len(ds.indexMergeHints) > 0 { - ds.indexMergeHints = nil - var msg string - if !isPossibleIdxMerge { - msg = "No available filter or available index." - } else if !sessionAndStmtPermission { - msg = "Got no_index_merge hint or tidb_enable_index_merge is off." - } else if ds.tableInfo.TempTableType == model.TempTableLocal { - msg = "Cannot use IndexMerge on temporary table." - } - msg = fmt.Sprintf("IndexMerge is inapplicable or disabled. %s", msg) - stmtCtx.AppendWarning(errors.Errorf(msg)) - logutil.BgLogger().Debug(msg) - } return ds.stats, nil } -func (ds *DataSource) generateAndPruneIndexMergePath(indexMergeConds []expression.Expression, needPrune bool) error { - regularPathCount := len(ds.possibleAccessPaths) - // 1. Generate possible IndexMerge paths for `OR`. - err := ds.generateIndexMergeOrPaths(indexMergeConds) - if err != nil { - return err - } - // 2. Generate possible IndexMerge paths for `AND`. - indexMergeAndPath := ds.generateIndexMergeAndPaths(regularPathCount) - if indexMergeAndPath != nil { - ds.possibleAccessPaths = append(ds.possibleAccessPaths, indexMergeAndPath) - } - - // 3. If needed, append a warning if no IndexMerge is generated. - - // If without hints, it means that `enableIndexMerge` is true - if len(ds.indexMergeHints) == 0 { - return nil - } - // With hints and without generated IndexMerge paths - if regularPathCount == len(ds.possibleAccessPaths) { - ds.indexMergeHints = nil - ds.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("IndexMerge is inapplicable")) - return nil - } - - // 4. If needPrune is true, prune non-IndexMerge paths. - - // Do not need to consider the regular paths in find_best_task(). - // So we can use index merge's row count as DataSource's row count. - if needPrune { - ds.possibleAccessPaths = ds.possibleAccessPaths[regularPathCount:] - minRowCount := ds.possibleAccessPaths[0].CountAfterAccess - for _, path := range ds.possibleAccessPaths { - if minRowCount < path.CountAfterAccess { - minRowCount = path.CountAfterAccess - } - } - if ds.stats.RowCount > minRowCount { - ds.stats = ds.tableStats.ScaleByExpectCnt(minRowCount) - } - } - return nil -} - // DeriveStats implements LogicalPlan DeriveStats interface. func (ts *LogicalTableScan) DeriveStats(_ []*property.StatsInfo, _ *expression.Schema, _ []*expression.Schema, _ [][]*expression.Column) (_ *property.StatsInfo, err error) { ts.Source.initStats(nil) @@ -562,337 +467,6 @@ func (is *LogicalIndexScan) DeriveStats(_ []*property.StatsInfo, selfSchema *exp return is.stats, nil } -// getIndexMergeOrPath generates all possible IndexMergeOrPaths. -func (ds *DataSource) generateIndexMergeOrPaths(filters []expression.Expression) error { - usedIndexCount := len(ds.possibleAccessPaths) - for i, cond := range filters { - sf, ok := cond.(*expression.ScalarFunction) - if !ok || sf.FuncName.L != ast.LogicOr { - continue - } - var partialPaths = make([]*util.AccessPath, 0, usedIndexCount) - dnfItems := expression.FlattenDNFConditions(sf) - for _, item := range dnfItems { - cnfItems := expression.SplitCNFItems(item) - itemPaths := ds.accessPathsForConds(cnfItems, usedIndexCount) - if len(itemPaths) == 0 { - partialPaths = nil - break - } - partialPath, err := ds.buildIndexMergePartialPath(itemPaths) - if err != nil { - return err - } - if partialPath == nil { - partialPaths = nil - break - } - partialPaths = append(partialPaths, partialPath) - } - // If all of the partialPaths use the same index, we will not use the indexMerge. - singlePath := true - for i := len(partialPaths) - 1; i >= 1; i-- { - if partialPaths[i].Index != partialPaths[i-1].Index { - singlePath = false - break - } - } - if singlePath { - continue - } - if len(partialPaths) > 1 { - possiblePath := ds.buildIndexMergeOrPath(filters, partialPaths, i) - if possiblePath == nil { - return nil - } - - accessConds := make([]expression.Expression, 0, len(partialPaths)) - for _, p := range partialPaths { - indexCondsForP := p.AccessConds[:] - indexCondsForP = append(indexCondsForP, p.IndexFilters...) - if len(indexCondsForP) > 0 { - accessConds = append(accessConds, expression.ComposeCNFCondition(ds.ctx, indexCondsForP...)) - } - } - accessDNF := expression.ComposeDNFCondition(ds.ctx, accessConds...) - sel, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, []expression.Expression{accessDNF}, nil) - if err != nil { - logutil.BgLogger().Debug("something wrong happened, use the default selectivity", zap.Error(err)) - sel = SelectionFactor - } - possiblePath.CountAfterAccess = sel * ds.tableStats.RowCount - ds.possibleAccessPaths = append(ds.possibleAccessPaths, possiblePath) - } - } - return nil -} - -// isInIndexMergeHints returns true if the input index name is not excluded by the IndexMerge hints, which means either -// (1) there's no IndexMerge hint, (2) there's IndexMerge hint but no specified index names, or (3) the input index -// name is specified in the IndexMerge hints. -func (ds *DataSource) isInIndexMergeHints(name string) bool { - if len(ds.indexMergeHints) == 0 { - return true - } - for _, hint := range ds.indexMergeHints { - if hint.indexHint == nil || len(hint.indexHint.IndexNames) == 0 { - return true - } - for _, hintName := range hint.indexHint.IndexNames { - if strings.EqualFold(strings.ToLower(name), strings.ToLower(hintName.String())) { - return true - } - } - } - return false -} - -// indexMergeHintsHasSpecifiedIdx returns true if there's IndexMerge hint, and it has specified index names. -func (ds *DataSource) indexMergeHintsHasSpecifiedIdx() bool { - for _, hint := range ds.indexMergeHints { - if hint.indexHint == nil || len(hint.indexHint.IndexNames) == 0 { - continue - } - if len(hint.indexHint.IndexNames) > 0 { - return true - } - } - return false -} - -// indexMergeHintsHasSpecifiedIdx return true if the input index name is specified in the IndexMerge hint. -func (ds *DataSource) isSpecifiedInIndexMergeHints(name string) bool { - for _, hint := range ds.indexMergeHints { - if hint.indexHint == nil || len(hint.indexHint.IndexNames) == 0 { - continue - } - for _, hintName := range hint.indexHint.IndexNames { - if strings.EqualFold(strings.ToLower(name), strings.ToLower(hintName.String())) { - return true - } - } - } - return false -} - -// accessPathsForConds generates all possible index paths for conditions. -func (ds *DataSource) accessPathsForConds(conditions []expression.Expression, usedIndexCount int) []*util.AccessPath { - var results = make([]*util.AccessPath, 0, usedIndexCount) - for i := 0; i < usedIndexCount; i++ { - path := &util.AccessPath{} - if ds.possibleAccessPaths[i].IsTablePath() { - if !ds.isInIndexMergeHints("primary") { - continue - } - if ds.tableInfo.IsCommonHandle { - path.IsCommonHandlePath = true - path.Index = ds.possibleAccessPaths[i].Index - } else { - path.IsIntHandlePath = true - } - err := ds.deriveTablePathStats(path, conditions, true) - if err != nil { - logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err)) - continue - } - var unsignedIntHandle bool - if path.IsIntHandlePath && ds.tableInfo.PKIsHandle { - if pkColInfo := ds.tableInfo.GetPkColInfo(); pkColInfo != nil { - unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.GetFlag()) - } - } - // If the path contains a full range, ignore it. - if ranger.HasFullRange(path.Ranges, unsignedIntHandle) { - continue - } - // If we have point or empty range, just remove other possible paths. - if len(path.Ranges) == 0 || path.OnlyPointRange(ds.SCtx()) { - if len(results) == 0 { - results = append(results, path) - } else { - results[0] = path - results = results[:1] - } - break - } - } else { - path.Index = ds.possibleAccessPaths[i].Index - if !ds.isInIndexMergeHints(path.Index.Name.L) { - continue - } - err := ds.fillIndexPath(path, conditions) - if err != nil { - logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err)) - continue - } - ds.deriveIndexPathStats(path, conditions, true) - // If the path contains a full range, ignore it. - if ranger.HasFullRange(path.Ranges, false) { - continue - } - // If we have empty range, or point range on unique index, just remove other possible paths. - if len(path.Ranges) == 0 || (path.OnlyPointRange(ds.SCtx()) && path.Index.Unique) { - if len(results) == 0 { - results = append(results, path) - } else { - results[0] = path - results = results[:1] - } - break - } - } - results = append(results, path) - } - return results -} - -// buildIndexMergePartialPath chooses the best index path from all possible paths. -// Now we choose the index with minimal estimate row count. -func (ds *DataSource) buildIndexMergePartialPath(indexAccessPaths []*util.AccessPath) (*util.AccessPath, error) { - if len(indexAccessPaths) == 1 { - return indexAccessPaths[0], nil - } - - minEstRowIndex := 0 - minEstRow := math.MaxFloat64 - for i := 0; i < len(indexAccessPaths); i++ { - rc := indexAccessPaths[i].CountAfterAccess - if len(indexAccessPaths[i].IndexFilters) > 0 { - rc = indexAccessPaths[i].CountAfterIndex - } - if rc < minEstRow { - minEstRowIndex = i - minEstRow = rc - } - } - return indexAccessPaths[minEstRowIndex], nil -} - -// buildIndexMergeOrPath generates one possible IndexMergePath. -func (ds *DataSource) buildIndexMergeOrPath(filters []expression.Expression, partialPaths []*util.AccessPath, current int) *util.AccessPath { - indexMergePath := &util.AccessPath{PartialIndexPaths: partialPaths} - indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[:current]...) - indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[current+1:]...) - var addCurrentFilter bool - for _, path := range partialPaths { - // If any partial path contains table filters, we need to keep the whole DNF filter in the Selection. - if len(path.TableFilters) > 0 { - addCurrentFilter = true - } - // If any partial path's index filter cannot be pushed to TiKV, we should keep the whole DNF filter. - if len(path.IndexFilters) != 0 && !expression.CanExprsPushDown(ds.ctx.GetSessionVars().StmtCtx, path.IndexFilters, ds.ctx.GetClient(), kv.TiKV) { - addCurrentFilter = true - // Clear IndexFilter, the whole filter will be put in indexMergePath.TableFilters. - path.IndexFilters = nil - } - if len(path.TableFilters) != 0 && !expression.CanExprsPushDown(ds.ctx.GetSessionVars().StmtCtx, path.TableFilters, ds.ctx.GetClient(), kv.TiKV) { - addCurrentFilter = true - path.TableFilters = nil - } - } - if addCurrentFilter { - indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[current]) - } - return indexMergePath -} - -// generateIndexMergeAndPaths generates IndexMerge paths for `AND` (a.k.a. intersection type IndexMerge) -func (ds *DataSource) generateIndexMergeAndPaths(normalPathCnt int) *util.AccessPath { - // For now, we only consider intersection type IndexMerge when the index names are specified in the hints. - if !ds.indexMergeHintsHasSpecifiedIdx() { - return nil - } - - // 1. Collect partial paths from normal paths. - var partialPaths []*util.AccessPath - for i := 0; i < normalPathCnt; i++ { - originalPath := ds.possibleAccessPaths[i] - // No need to consider table path as a partial path. - if ds.possibleAccessPaths[i].IsTablePath() { - continue - } - if !ds.isSpecifiedInIndexMergeHints(originalPath.Index.Name.L) { - continue - } - // If the path contains a full range, ignore it. - if ranger.HasFullRange(originalPath.Ranges, false) { - continue - } - newPath := originalPath.Clone() - partialPaths = append(partialPaths, newPath) - } - if len(partialPaths) < 2 { - return nil - } - - // 2. Collect filters that can't be covered by the partial paths and deduplicate them. - finalFilters := make([]expression.Expression, 0) - partialFilters := make([]expression.Expression, 0, len(partialPaths)) - hashCodeSet := make(map[string]struct{}) - for _, path := range partialPaths { - // Classify filters into coveredConds and notCoveredConds. - coveredConds := make([]expression.Expression, 0, len(path.AccessConds)+len(path.IndexFilters)) - notCoveredConds := make([]expression.Expression, 0, len(path.IndexFilters)+len(path.TableFilters)) - // AccessConds can be covered by partial path. - coveredConds = append(coveredConds, path.AccessConds...) - for i, cond := range path.IndexFilters { - // IndexFilters can be covered by partial path if it can be pushed down to TiKV. - if !expression.CanExprsPushDown(ds.ctx.GetSessionVars().StmtCtx, []expression.Expression{cond}, ds.ctx.GetClient(), kv.TiKV) { - path.IndexFilters = append(path.IndexFilters[:i], path.IndexFilters[i+1:]...) - notCoveredConds = append(notCoveredConds, cond) - } else { - coveredConds = append(coveredConds, cond) - } - } - // TableFilters can't be covered by partial path. - notCoveredConds = append(notCoveredConds, path.TableFilters...) - - // Record covered filters in hashCodeSet. - // Note that we only record filters that not appear in the notCoveredConds. It's possible that a filter appear - // in both coveredConds and notCoveredConds (e.g. because of prefix index). So we need this extra check to - // avoid wrong deduplication. - notCoveredHashCodeSet := make(map[string]struct{}) - for _, cond := range notCoveredConds { - hashCode := string(cond.HashCode(ds.ctx.GetSessionVars().StmtCtx)) - notCoveredHashCodeSet[hashCode] = struct{}{} - } - for _, cond := range coveredConds { - hashCode := string(cond.HashCode(ds.ctx.GetSessionVars().StmtCtx)) - if _, ok := notCoveredHashCodeSet[hashCode]; !ok { - hashCodeSet[hashCode] = struct{}{} - } - } - - finalFilters = append(finalFilters, notCoveredConds...) - partialFilters = append(partialFilters, coveredConds...) - } - - // Remove covered filters from finalFilters and deduplicate finalFilters. - dedupedFinalFilters := make([]expression.Expression, 0, len(finalFilters)) - for _, cond := range finalFilters { - hashCode := string(cond.HashCode(ds.ctx.GetSessionVars().StmtCtx)) - if _, ok := hashCodeSet[hashCode]; !ok { - dedupedFinalFilters = append(dedupedFinalFilters, cond) - hashCodeSet[hashCode] = struct{}{} - } - } - - // 3. Estimate the row count after partial paths. - sel, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, partialFilters, nil) - if err != nil { - logutil.BgLogger().Debug("something wrong happened, use the default selectivity", zap.Error(err)) - sel = SelectionFactor - } - - indexMergePath := &util.AccessPath{ - PartialIndexPaths: partialPaths, - IndexMergeIsIntersection: true, - TableFilters: dedupedFinalFilters, - CountAfterAccess: sel * ds.tableStats.RowCount, - } - return indexMergePath -} - // DeriveStats implement LogicalPlan DeriveStats interface. func (p *LogicalSelection) DeriveStats(childStats []*property.StatsInfo, _ *expression.Schema, _ []*expression.Schema, _ [][]*expression.Column) (*property.StatsInfo, error) { if p.stats != nil { diff --git a/planner/core/task.go b/planner/core/task.go index cc27029d83c8e..99952038688fe 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1239,12 +1239,17 @@ func CheckAggCanPushCop(sctx sessionctx.Context, aggFuncs []*aggregation.AggFunc ret = false } - if !ret && sc.InExplainStmt { + if !ret { storageName := storeType.Name() if storeType == kv.UnSpecified { storageName = "storage layer" } - sc.AppendWarning(errors.New("Aggregation can not be pushed to " + storageName + " because " + reason)) + warnErr := errors.New("Aggregation can not be pushed to " + storageName + " because " + reason) + if sc.InExplainStmt { + sc.AppendWarning(warnErr) + } else { + sc.AppendExtraWarning(warnErr) + } } return ret } diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 14c04c6cfb0ab..c1dbae406a922 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -1969,7 +1969,8 @@ " └─TableRangeScan_8 3333.33 923531.15 cop[tikv] table:t range:(1,+inf], keep order:false, stats:pseudo" ], "Warnings": [ - "Note 1105 [t] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}" + "Note 1105 [t] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}", + "Note 1105 [t,f,f_g] remain after pruning paths for t given Prop{SortItems: [{test.t.f asc}], TaskTp: rootTask}" ] }, { @@ -2014,7 +2015,8 @@ " └─TableRowIDScan_12(Probe) 10.00 2770.59 cop[tikv] table:t keep order:false, stats:pseudo" ], "Warnings": [ - "Note 1105 [t,g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}" + "Note 1105 [t,g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}", + "Note 1105 [t,f_g,g] remain after pruning paths for t given Prop{SortItems: [{test.t.f asc}], TaskTp: rootTask}" ] }, { @@ -2026,6 +2028,7 @@ "└─TableRowIDScan_13(Probe) 10.00 2770.59 cop[tikv] table:t keep order:false, stats:pseudo" ], "Warnings": [ + "Note 1105 [t] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}", "Note 1105 [t,c_d_e] remain after pruning paths for t given Prop{SortItems: [{test.t.c asc} {test.t.e asc}], TaskTp: rootTask}" ] } @@ -6226,9 +6229,9 @@ { "SQL": "explain format = 'brief' select /*+ qb_name(qb, v12) read_from_storage(tiflash[t1@qb, t@qb]), shuffle_join(t1@qb, t@qb) */ * from v12;", "Plan": [ - "Projection 12500.00 root test.t.a, test.t.b", - "└─TableReader 12500.00 root data:ExchangeSender", - " └─ExchangeSender 12500.00 mpp[tiflash] ExchangeType: PassThrough", + "TableReader 12500.00 root data:ExchangeSender", + "└─ExchangeSender 12500.00 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 12500.00 mpp[tiflash] test.t.a, test.t.b", " └─HashJoin 12500.00 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", " ├─ExchangeReceiver(Build) 10000.00 mpp[tiflash] ", " │ └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", diff --git a/planner/core/testdata/plan_suite_in.json b/planner/core/testdata/plan_suite_in.json index c38250c802454..d433f5dd88dbe 100644 --- a/planner/core/testdata/plan_suite_in.json +++ b/planner/core/testdata/plan_suite_in.json @@ -2,10 +2,70 @@ { "name": "TestMPPHints", "cases": [ + "select /*+ MPP_1PHASE_AGG() */ a, sum(b) from t group by a, c", + "select /*+ MPP_2PHASE_AGG() */ a, sum(b) from t group by a, c", + "select /*+ shuffle_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "select /*+ broadcast_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + + // READ_FROM_STORAGE hint "select /*+ read_from_storage(tiflash[t]), MPP_1PHASE_AGG() */ a, sum(b) from t group by a, c", "select /*+ read_from_storage(tiflash[t]), MPP_2PHASE_AGG() */ a, sum(b) from t group by a, c", "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", - "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a" + "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + + // Join hint + "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2), hash_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2), hash_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + + "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2), hash_join_build(t1) */ * from t t1, t t2 where t1.a=t2.a", + "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2), hash_join_build(t2) */ * from t t1, t t2 where t1.a=t2.a", + + "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2), hash_join_probe(t1) */ * from t t1, t t2 where t1.a=t2.a", + "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2), hash_join_probe(t2) */ * from t t1, t t2 where t1.a=t2.a", + + "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2), merge_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2), merge_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + + "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2), INL_JOIN(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2), INL_JOIN(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + + // AGG hint + "select /*+ read_from_storage(tiflash[t]), MPP_1PHASE_AGG(), hash_agg() */ a, sum(b) from t group by a, c", + "select /*+ read_from_storage(tiflash[t]), MPP_2PHASE_AGG(), stream_agg() */ a, sum(b) from t group by a, c", + + // Index hint + "select /*+ read_from_storage(tiflash[t]), MPP_1PHASE_AGG(), use_index(t, idx_a) */ a, sum(b) from t where a > 1 group by a, c", + "select /*+ read_from_storage(tiflash[t]), MPP_1PHASE_AGG(), ignore_index(t, idx_a) */ a, sum(b) from t where a > 1 group by a, c", + "select /*+ read_from_storage(tiflash[t]), MPP_2PHASE_AGG(), force_index(t, idx_b) */ a, sum(b) from t where b < 2 group by a, c", + "select /*+ read_from_storage(tiflash[t]), MPP_2PHASE_AGG(), index_merge(t, idx_b, idx_a) */ a, sum(b) from t where b < 2 or a > 2 group by a, c", + + // Join Order hint + "select /*+ read_from_storage(tiflash[t1, t2, t3]), shuffle_join(t1, t2, t3), straight_join() */ * from t t1, t t2, t t3 where t1.a=t2.a and t2.b=t3.b", + "select /*+ read_from_storage(tiflash[t1, t2, t3]), shuffle_join(t1, t2, t3), leading(t3, t1) */ * from t t1, t t2, t t3 where t1.a=t2.a and t2.b=t3.b", + "select /*+ read_from_storage(tiflash[t1, t2, t3]), broadcast_join(t1, t2, t3), straight_join() */ * from t t2, t t1, t t3 where t1.a=t2.a and t2.b=t3.b", + "select /*+ read_from_storage(tiflash[t1, t2, t3]), broadcast_join(t1, t2, t3), leading(t2, t3) */ * from t t1, t t2, t t3 where t1.a=t2.a and t2.b=t3.b", + + // View Hint + "select /*+ qb_name(qb, v), MPP_1PHASE_AGG(@qb) */ * from v", + "select /*+ qb_name(qb, v), MPP_2PHASE_AGG(@qb) */ * from v", + "select /*+ qb_name(qb, v1), shuffle_join(t1@qb, t2@qb) */ * from v1", + "select /*+ qb_name(qb, v1), broadcast_join(t1@qb, t2@qb) */ * from v1", + + // Subquery hint + "SELECT /*+ shuffle_join(t) */ * FROM t WHERE EXISTS (SELECT /*+ SEMI_JOIN_REWRITE */ 1 FROM t t1 WHERE t1.b = t.b);", + "SELECT /*+ broadcast_join(t) */ * FROM t WHERE EXISTS (SELECT /*+ SEMI_JOIN_REWRITE */ 1 FROM t t1 WHERE t1.b = t.b);", + "select * from t t1 where t1.a < (select /*+ MPP_1PHASE_AGG() */ sum(t2.a) from t t2 where t2.b = t1.b);", + "select * from t t1 where t1.a < (select /*+ MPP_2PHASE_AGG() */ sum(t2.a) from t t2 where t2.b = t1.b);", + + // CTE + "WITH CTE AS (SELECT /*+ MPP_1PHASE_AGG() */ count(*) as a, b FROM t WHERE t.a < 60 group by b) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "WITH CTE AS (SELECT /*+ MPP_2PHASE_AGG() */ count(*) as a, b FROM t WHERE t.a < 60 group by b) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "WITH CTE AS (SELECT /*+ shuffle_join(t1, t) */ t.a, t.b FROM t join t t1 where t.a = t1.a) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "WITH CTE AS (SELECT /*+ broadcast_join(t1, t) */ t.a, t.b FROM t join t t1 where t.a = t1.a) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "WITH CTE AS (SELECT /*+ MERGE(), MPP_1PHASE_AGG() */ count(*) as a, b FROM t WHERE t.a < 60 group by b) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "WITH CTE AS (SELECT /*+ MERGE(), MPP_2PHASE_AGG() */ count(*) as a, b FROM t WHERE t.a < 60 group by b) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "WITH CTE AS (SELECT /*+ MERGE(), shuffle_join(t1, t) */ t.a, t.b FROM t join t t1 where t.a = t1.a) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "WITH CTE AS (SELECT /*+ MERGE(), broadcast_join(t1, t) */ t.a, t.b FROM t join t t1 where t.a = t1.a) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;" ] }, { diff --git a/planner/core/testdata/plan_suite_out.json b/planner/core/testdata/plan_suite_out.json index b3a7664b2b2fd..31964823e95f2 100644 --- a/planner/core/testdata/plan_suite_out.json +++ b/planner/core/testdata/plan_suite_out.json @@ -2,6 +2,69 @@ { "Name": "TestMPPHints", "Cases": [ + { + "SQL": "select /*+ MPP_1PHASE_AGG() */ a, sum(b) from t group by a, c", + "Plan": [ + "TableReader 8000.00 root data:ExchangeSender", + "└─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 8000.00 mpp[tiflash] test.t.a, Column#5", + " └─Projection 8000.00 mpp[tiflash] Column#5, test.t.a", + " └─HashAgg 8000.00 mpp[tiflash] group by:Column#10, Column#11, funcs:sum(Column#8)->Column#5, funcs:firstrow(Column#9)->test.t.a", + " └─Projection 10000.00 mpp[tiflash] cast(test.t.b, decimal(10,0) BINARY)->Column#8, test.t.a, test.t.a, test.t.c", + " └─ExchangeReceiver 10000.00 mpp[tiflash] ", + " └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary], [name: test.t.c, collate: binary]", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ MPP_2PHASE_AGG() */ a, sum(b) from t group by a, c", + "Plan": [ + "TableReader 8000.00 root data:ExchangeSender", + "└─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 8000.00 mpp[tiflash] test.t.a, Column#5", + " └─Projection 8000.00 mpp[tiflash] Column#5, test.t.a", + " └─HashAgg 8000.00 mpp[tiflash] group by:test.t.a, test.t.c, funcs:sum(Column#10)->Column#5, funcs:firstrow(test.t.a)->test.t.a", + " └─ExchangeReceiver 8000.00 mpp[tiflash] ", + " └─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary], [name: test.t.c, collate: binary]", + " └─HashAgg 8000.00 mpp[tiflash] group by:Column#13, Column#14, funcs:sum(Column#12)->Column#10", + " └─Projection 10000.00 mpp[tiflash] cast(test.t.b, decimal(10,0) BINARY)->Column#12, test.t.a, test.t.c", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ shuffle_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 9990.00 mpp[tiflash] ", + " └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ broadcast_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": null + }, { "SQL": "select /*+ read_from_storage(tiflash[t]), MPP_1PHASE_AGG() */ a, sum(b) from t group by a, c", "Plan": [ @@ -14,7 +77,8 @@ " └─ExchangeReceiver 10000.00 mpp[tiflash] ", " └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary], [name: test.t.c, collate: binary]", " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" - ] + ], + "Warn": null }, { "SQL": "select /*+ read_from_storage(tiflash[t]), MPP_2PHASE_AGG() */ a, sum(b) from t group by a, c", @@ -29,7 +93,8 @@ " └─HashAgg 8000.00 mpp[tiflash] group by:Column#13, Column#14, funcs:sum(Column#12)->Column#10", " └─Projection 10000.00 mpp[tiflash] cast(test.t.b, decimal(10,0) BINARY)->Column#12, test.t.a, test.t.c", " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" - ] + ], + "Warn": null }, { "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", @@ -45,7 +110,8 @@ " └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", " └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" - ] + ], + "Warn": null }, { "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", @@ -59,6 +125,732 @@ " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2), hash_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]Join hints are conflict, you can only specify one type of join" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2), hash_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]Join hints are conflict, you can only specify one type of join" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2), hash_join_build(t1) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]Join hints are conflict, you can only specify one type of join" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2), hash_join_build(t2) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]Join hints are conflict, you can only specify one type of join" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2), hash_join_probe(t1) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]Join hints are conflict, you can only specify one type of join" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2), hash_join_probe(t2) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]Join hints are conflict, you can only specify one type of join" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2), merge_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]Join hints are conflict, you can only specify one type of join" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2), merge_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]Join hints are conflict, you can only specify one type of join" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), shuffle_join(t1, t2), INL_JOIN(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]Join hints are conflict, you can only specify one type of join" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2]), broadcast_join(t1, t2), INL_JOIN(t1, t2) */ * from t t1, t t2 where t1.a=t2.a", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]Join hints are conflict, you can only specify one type of join" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t]), MPP_1PHASE_AGG(), hash_agg() */ a, sum(b) from t group by a, c", + "Plan": [ + "TableReader 8000.00 root data:ExchangeSender", + "└─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 8000.00 mpp[tiflash] test.t.a, Column#5", + " └─Projection 8000.00 mpp[tiflash] Column#5, test.t.a", + " └─HashAgg 8000.00 mpp[tiflash] group by:Column#10, Column#11, funcs:sum(Column#8)->Column#5, funcs:firstrow(Column#9)->test.t.a", + " └─Projection 10000.00 mpp[tiflash] cast(test.t.b, decimal(10,0) BINARY)->Column#8, test.t.a, test.t.a, test.t.c", + " └─ExchangeReceiver 10000.00 mpp[tiflash] ", + " └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary], [name: test.t.c, collate: binary]", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t]), MPP_2PHASE_AGG(), stream_agg() */ a, sum(b) from t group by a, c", + "Plan": [ + "TableReader 8000.00 root data:ExchangeSender", + "└─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 8000.00 mpp[tiflash] test.t.a, Column#5", + " └─Projection 8000.00 mpp[tiflash] Column#5, test.t.a", + " └─HashAgg 8000.00 mpp[tiflash] group by:test.t.a, test.t.c, funcs:sum(Column#6)->Column#5, funcs:firstrow(test.t.a)->test.t.a", + " └─ExchangeReceiver 8000.00 mpp[tiflash] ", + " └─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary], [name: test.t.c, collate: binary]", + " └─HashAgg 8000.00 mpp[tiflash] group by:Column#10, Column#9, funcs:sum(Column#8)->Column#6", + " └─Projection 10000.00 mpp[tiflash] cast(test.t.b, decimal(10,0) BINARY)->Column#8, test.t.a, test.t.c", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t]), MPP_1PHASE_AGG(), use_index(t, idx_a) */ a, sum(b) from t where a > 1 group by a, c", + "Plan": [ + "Projection 2666.67 root test.t.a, Column#5", + "└─HashAgg 2666.67 root group by:test.t.a, test.t.c, funcs:sum(Column#7)->Column#5, funcs:firstrow(test.t.a)->test.t.a", + " └─IndexLookUp 2666.67 root ", + " ├─IndexRangeScan(Build) 3333.33 cop[tikv] table:t, index:idx_a(a) range:(1,+inf], keep order:false, stats:pseudo", + " └─HashAgg(Probe) 2666.67 cop[tikv] group by:test.t.a, test.t.c, funcs:sum(test.t.b)->Column#7", + " └─TableRowIDScan 3333.33 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]No available path for table test.t with the store type tiflash of the hint /*+ read_from_storage */, please check the status of the table replica and variable value of tidb_isolation_read_engines(map[0:{} 1:{} 2:{}])", + "[planner:1815]The agg can not push down to the MPP side, the MPP_1PHASE_AGG() hint is invalid" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t]), MPP_1PHASE_AGG(), ignore_index(t, idx_a) */ a, sum(b) from t where a > 1 group by a, c", + "Plan": [ + "TableReader 2666.67 root data:ExchangeSender", + "└─ExchangeSender 2666.67 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 2666.67 mpp[tiflash] test.t.a, Column#5", + " └─Projection 2666.67 mpp[tiflash] Column#5, test.t.a", + " └─HashAgg 2666.67 mpp[tiflash] group by:Column#10, Column#11, funcs:sum(Column#8)->Column#5, funcs:firstrow(Column#9)->test.t.a", + " └─Projection 3333.33 mpp[tiflash] cast(test.t.b, decimal(10,0) BINARY)->Column#8, test.t.a, test.t.a, test.t.c", + " └─ExchangeReceiver 3333.33 mpp[tiflash] ", + " └─ExchangeSender 3333.33 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary], [name: test.t.c, collate: binary]", + " └─Selection 3333.33 mpp[tiflash] gt(test.t.a, 1)", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t]), MPP_2PHASE_AGG(), force_index(t, idx_b) */ a, sum(b) from t where b < 2 group by a, c", + "Plan": [ + "Projection 2658.67 root test.t.a, Column#5", + "└─HashAgg 2658.67 root group by:test.t.a, test.t.c, funcs:sum(Column#7)->Column#5, funcs:firstrow(test.t.a)->test.t.a", + " └─IndexLookUp 2658.67 root ", + " ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t, index:idx_b(b) range:[-inf,2), keep order:false, stats:pseudo", + " └─HashAgg(Probe) 2658.67 cop[tikv] group by:test.t.a, test.t.c, funcs:sum(test.t.b)->Column#7", + " └─TableRowIDScan 3323.33 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]No available path for table test.t with the store type tiflash of the hint /*+ read_from_storage */, please check the status of the table replica and variable value of tidb_isolation_read_engines(map[0:{} 1:{} 2:{}])", + "[planner:1815]The agg can not push down to the MPP side, the MPP_2PHASE_AGG() hint is invalid" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t]), MPP_2PHASE_AGG(), index_merge(t, idx_b, idx_a) */ a, sum(b) from t where b < 2 or a > 2 group by a, c", + "Plan": [ + "TableReader 4439.11 root data:ExchangeSender", + "└─ExchangeSender 4439.11 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 4439.11 mpp[tiflash] test.t.a, Column#5", + " └─Projection 4439.11 mpp[tiflash] Column#5, test.t.a", + " └─HashAgg 4439.11 mpp[tiflash] group by:test.t.a, test.t.c, funcs:sum(Column#13)->Column#5, funcs:firstrow(test.t.a)->test.t.a", + " └─ExchangeReceiver 4439.11 mpp[tiflash] ", + " └─ExchangeSender 4439.11 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary], [name: test.t.c, collate: binary]", + " └─HashAgg 4439.11 mpp[tiflash] group by:Column#16, Column#17, funcs:sum(Column#15)->Column#13", + " └─Projection 5548.89 mpp[tiflash] cast(test.t.b, decimal(10,0) BINARY)->Column#15, test.t.a, test.t.c", + " └─Selection 5548.89 mpp[tiflash] or(lt(test.t.b, 2), gt(test.t.a, 2))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": [ + "[parser:8061]Optimizer hint index_merge is not supported by TiDB and is ignored" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2, t3]), shuffle_join(t1, t2, t3), straight_join() */ * from t t1, t t2, t t3 where t1.a=t2.a and t2.b=t3.b", + "Plan": [ + "TableReader 15593.77 root data:ExchangeSender", + "└─ExchangeSender 15593.77 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 15593.77 mpp[tiflash] inner join, equal:[eq(test.t.b, test.t.b)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t3 keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 12475.01 mpp[tiflash] ", + " └─ExchangeSender 12475.01 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " └─HashJoin 12475.01 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9980.01 mpp[tiflash] ", + " │ └─ExchangeSender 9980.01 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " │ └─Selection 9980.01 mpp[tiflash] not(isnull(test.t.a)), not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 9990.00 mpp[tiflash] ", + " └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2, t3]), shuffle_join(t1, t2, t3), leading(t3, t1) */ * from t t1, t t2, t t3 where t1.a=t2.a and t2.b=t3.b", + "Plan": [ + "TableReader 15593.77 root data:ExchangeSender", + "└─ExchangeSender 15593.77 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 15593.77 mpp[tiflash] inner join, equal:[eq(test.t.b, test.t.b)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t3 keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 12475.01 mpp[tiflash] ", + " └─ExchangeSender 12475.01 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " └─HashJoin 12475.01 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9980.01 mpp[tiflash] ", + " │ └─ExchangeSender 9980.01 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " │ └─Selection 9980.01 mpp[tiflash] not(isnull(test.t.a)), not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 9990.00 mpp[tiflash] ", + " └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]leading hint is inapplicable, check the join type or the join algorithm hint", + "[planner:1815]leading hint is inapplicable, check the join type or the join algorithm hint" + ] + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2, t3]), broadcast_join(t1, t2, t3), straight_join() */ * from t t2, t t1, t t3 where t1.a=t2.a and t2.b=t3.b", + "Plan": [ + "TableReader 15593.77 root data:ExchangeSender", + "└─ExchangeSender 15593.77 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 15593.77 mpp[tiflash] inner join, equal:[eq(test.t.b, test.t.b)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t3 keep order:false, stats:pseudo", + " └─HashJoin(Probe) 12475.01 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9980.01 mpp[tiflash] ", + " │ └─ExchangeSender 9980.01 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9980.01 mpp[tiflash] not(isnull(test.t.a)), not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ read_from_storage(tiflash[t1, t2, t3]), broadcast_join(t1, t2, t3), leading(t2, t3) */ * from t t1, t t2, t t3 where t1.a=t2.a and t2.b=t3.b", + "Plan": [ + "TableReader 15593.77 root data:ExchangeSender", + "└─ExchangeSender 15593.77 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 15593.77 mpp[tiflash] inner join, equal:[eq(test.t.b, test.t.b)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t3 keep order:false, stats:pseudo", + " └─HashJoin(Probe) 12475.01 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9980.01 mpp[tiflash] ", + " │ └─ExchangeSender 9980.01 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9980.01 mpp[tiflash] not(isnull(test.t.a)), not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]leading hint is inapplicable, check the join type or the join algorithm hint", + "[planner:1815]leading hint is inapplicable, check the join type or the join algorithm hint" + ] + }, + { + "SQL": "select /*+ qb_name(qb, v), MPP_1PHASE_AGG(@qb) */ * from v", + "Plan": [ + "TableReader 8000.00 root data:ExchangeSender", + "└─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 8000.00 mpp[tiflash] test.t.a, Column#5", + " └─Projection 8000.00 mpp[tiflash] Column#5, test.t.a", + " └─HashAgg 8000.00 mpp[tiflash] group by:Column#10, Column#11, funcs:sum(Column#8)->Column#5, funcs:firstrow(Column#9)->test.t.a", + " └─Projection 10000.00 mpp[tiflash] cast(test.t.b, decimal(10,0) BINARY)->Column#8, test.t.a, test.t.a, test.t.c", + " └─ExchangeReceiver 10000.00 mpp[tiflash] ", + " └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary], [name: test.t.c, collate: binary]", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ qb_name(qb, v), MPP_2PHASE_AGG(@qb) */ * from v", + "Plan": [ + "TableReader 8000.00 root data:ExchangeSender", + "└─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 8000.00 mpp[tiflash] test.t.a, Column#5", + " └─Projection 8000.00 mpp[tiflash] Column#5, test.t.a", + " └─HashAgg 8000.00 mpp[tiflash] group by:test.t.a, test.t.c, funcs:sum(Column#10)->Column#5, funcs:firstrow(test.t.a)->test.t.a", + " └─ExchangeReceiver 8000.00 mpp[tiflash] ", + " └─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary], [name: test.t.c, collate: binary]", + " └─HashAgg 8000.00 mpp[tiflash] group by:Column#13, Column#14, funcs:sum(Column#12)->Column#10", + " └─Projection 10000.00 mpp[tiflash] cast(test.t.b, decimal(10,0) BINARY)->Column#12, test.t.a, test.t.c", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ qb_name(qb, v1), shuffle_join(t1@qb, t2@qb) */ * from v1", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 12487.50 mpp[tiflash] test.t.a", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 9990.00 mpp[tiflash] ", + " └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select /*+ qb_name(qb, v1), broadcast_join(t1@qb, t2@qb) */ * from v1", + "Plan": [ + "TableReader 12487.50 root data:ExchangeSender", + "└─ExchangeSender 12487.50 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 12487.50 mpp[tiflash] test.t.a", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "SELECT /*+ shuffle_join(t) */ * FROM t WHERE EXISTS (SELECT /*+ SEMI_JOIN_REWRITE */ 1 FROM t t1 WHERE t1.b = t.b);", + "Plan": [ + "TableReader 7992.00 root data:ExchangeSender", + "└─ExchangeSender 7992.00 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 7992.00 mpp[tiflash] semi join, equal:[eq(test.t.b, test.t.b)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]There are no matching table names for (t) in optimizer hint /*+ SHUFFLE_JOIN(t) */ or /*+ SHUFFLE_JOIN(t) */. Maybe you can use the table alias name", + "[parser:1064]Optimizer hint syntax error at line 1 column 109 near \"\" " + ] + }, + { + "SQL": "SELECT /*+ broadcast_join(t) */ * FROM t WHERE EXISTS (SELECT /*+ SEMI_JOIN_REWRITE */ 1 FROM t t1 WHERE t1.b = t.b);", + "Plan": [ + "TableReader 7992.00 root data:ExchangeSender", + "└─ExchangeSender 7992.00 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 7992.00 mpp[tiflash] semi join, equal:[eq(test.t.b, test.t.b)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]There are no matching table names for (t) in optimizer hint /*+ BROADCAST_JOIN(t) */ or /*+ TIDB_BCJ(t) */. Maybe you can use the table alias name", + "[parser:1064]Optimizer hint syntax error at line 1 column 111 near \"\" " + ] + }, + { + "SQL": "select * from t t1 where t1.a < (select /*+ MPP_1PHASE_AGG() */ sum(t2.a) from t t2 where t2.b = t1.b);", + "Plan": [ + "TableReader 9990.00 root data:ExchangeSender", + "└─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 9990.00 mpp[tiflash] test.t.a, test.t.b, test.t.c", + " └─HashJoin 9990.00 mpp[tiflash] inner join, equal:[eq(test.t.b, test.t.b)], other cond:lt(cast(test.t.a, decimal(10,0) BINARY), Column#9)", + " ├─ExchangeReceiver(Build) 7992.00 mpp[tiflash] ", + " │ └─ExchangeSender 7992.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Projection 7992.00 mpp[tiflash] Column#9, test.t.b", + " │ └─HashAgg 7992.00 mpp[tiflash] group by:Column#26, funcs:sum(Column#24)->Column#9, funcs:firstrow(Column#25)->test.t.b", + " │ └─Projection 9990.00 mpp[tiflash] cast(test.t.a, decimal(10,0) BINARY)->Column#24, test.t.b, test.t.b", + " │ └─ExchangeReceiver 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "select * from t t1 where t1.a < (select /*+ MPP_2PHASE_AGG() */ sum(t2.a) from t t2 where t2.b = t1.b);", + "Plan": [ + "TableReader 9990.00 root data:ExchangeSender", + "└─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 9990.00 mpp[tiflash] test.t.a, test.t.b, test.t.c", + " └─HashJoin 9990.00 mpp[tiflash] inner join, equal:[eq(test.t.b, test.t.b)], other cond:lt(cast(test.t.a, decimal(10,0) BINARY), Column#9)", + " ├─ExchangeReceiver(Build) 7992.00 mpp[tiflash] ", + " │ └─ExchangeSender 7992.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Projection 7992.00 mpp[tiflash] Column#9, test.t.b", + " │ └─HashAgg 7992.00 mpp[tiflash] group by:test.t.b, funcs:sum(Column#13)->Column#9, funcs:firstrow(test.t.b)->test.t.b", + " │ └─ExchangeReceiver 7992.00 mpp[tiflash] ", + " │ └─ExchangeSender 7992.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " │ └─HashAgg 7992.00 mpp[tiflash] group by:Column#29, funcs:sum(Column#28)->Column#13", + " │ └─Projection 9990.00 mpp[tiflash] cast(test.t.a, decimal(10,0) BINARY)->Column#28, test.t.b", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.b))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "WITH CTE AS (SELECT /*+ MPP_1PHASE_AGG() */ count(*) as a, b FROM t WHERE t.a < 60 group by b) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "Plan": [ + "HashAgg 3403.09 root group by:Column#10, Column#11, funcs:firstrow(Column#10)->Column#10, funcs:firstrow(Column#11)->Column#11", + "└─Union 3403.09 root ", + " ├─Selection 1701.55 root lt(Column#6, 18)", + " │ └─CTEFullScan 2126.93 root CTE:cte data:CTE_0", + " └─Selection 1701.55 root gt(test.t.b, 1)", + " └─CTEFullScan 2126.93 root CTE:cte data:CTE_0", + "CTE_0 2126.93 root Non-Recursive CTE", + "└─TableReader(Seed Part) 2126.93 root data:ExchangeSender", + " └─ExchangeSender 2126.93 mpp[tiflash] ExchangeType: PassThrough", + " └─Selection 2126.93 mpp[tiflash] or(lt(Column#5, 18), gt(test.t.b, 1))", + " └─Projection 2658.67 mpp[tiflash] Column#5, test.t.b", + " └─HashAgg 2658.67 mpp[tiflash] group by:test.t.b, funcs:count(1)->Column#5, funcs:firstrow(test.t.b)->test.t.b", + " └─ExchangeReceiver 3323.33 mpp[tiflash] ", + " └─ExchangeSender 3323.33 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " └─Selection 3323.33 mpp[tiflash] lt(test.t.a, 60)", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "WITH CTE AS (SELECT /*+ MPP_2PHASE_AGG() */ count(*) as a, b FROM t WHERE t.a < 60 group by b) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "Plan": [ + "HashAgg 3403.09 root group by:Column#10, Column#11, funcs:firstrow(Column#10)->Column#10, funcs:firstrow(Column#11)->Column#11", + "└─Union 3403.09 root ", + " ├─Selection 1701.55 root lt(Column#6, 18)", + " │ └─CTEFullScan 2126.93 root CTE:cte data:CTE_0", + " └─Selection 1701.55 root gt(test.t.b, 1)", + " └─CTEFullScan 2126.93 root CTE:cte data:CTE_0", + "CTE_0 2126.93 root Non-Recursive CTE", + "└─TableReader(Seed Part) 2126.93 root data:ExchangeSender", + " └─ExchangeSender 2126.93 mpp[tiflash] ExchangeType: PassThrough", + " └─Selection 2126.93 mpp[tiflash] or(lt(Column#5, 18), gt(test.t.b, 1))", + " └─Projection 2658.67 mpp[tiflash] Column#5, test.t.b", + " └─HashAgg 2658.67 mpp[tiflash] group by:test.t.b, funcs:sum(Column#22)->Column#5, funcs:firstrow(test.t.b)->test.t.b", + " └─ExchangeReceiver 2658.67 mpp[tiflash] ", + " └─ExchangeSender 2658.67 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " └─HashAgg 2658.67 mpp[tiflash] group by:test.t.b, funcs:count(1)->Column#22", + " └─Selection 3323.33 mpp[tiflash] lt(test.t.a, 60)", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "WITH CTE AS (SELECT /*+ shuffle_join(t1, t) */ t.a, t.b FROM t join t t1 where t.a = t1.a) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "Plan": [ + "HashAgg 7095.48 root group by:Column#13, Column#14, funcs:firstrow(Column#13)->Column#13, funcs:firstrow(Column#14)->Column#14", + "└─Union 11086.68 root ", + " ├─Selection 5543.34 root lt(test.t.a, 18)", + " │ └─CTEFullScan 6929.18 root CTE:cte data:CTE_0", + " └─Selection 5543.34 root gt(test.t.b, 1)", + " └─CTEFullScan 6929.18 root CTE:cte data:CTE_0", + "CTE_0 6929.18 root Non-Recursive CTE", + "└─TableReader(Seed Part) 6929.18 root data:ExchangeSender", + " └─ExchangeSender 6929.18 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 6929.18 mpp[tiflash] test.t.a, test.t.b", + " └─HashJoin 6929.18 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)], other cond:or(lt(test.t.a, 18), gt(test.t.b, 1))", + " ├─ExchangeReceiver(Build) 5543.34 mpp[tiflash] ", + " │ └─ExchangeSender 5543.34 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " │ └─Selection 5543.34 mpp[tiflash] not(isnull(test.t.a)), or(lt(test.t.a, 18), gt(test.t.b, 1))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 9990.00 mpp[tiflash] ", + " └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "WITH CTE AS (SELECT /*+ broadcast_join(t1, t) */ t.a, t.b FROM t join t t1 where t.a = t1.a) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "Plan": [ + "HashAgg 7095.48 root group by:Column#13, Column#14, funcs:firstrow(Column#13)->Column#13, funcs:firstrow(Column#14)->Column#14", + "└─Union 11086.68 root ", + " ├─Selection 5543.34 root lt(test.t.a, 18)", + " │ └─CTEFullScan 6929.18 root CTE:cte data:CTE_0", + " └─Selection 5543.34 root gt(test.t.b, 1)", + " └─CTEFullScan 6929.18 root CTE:cte data:CTE_0", + "CTE_0 6929.18 root Non-Recursive CTE", + "└─TableReader(Seed Part) 6929.18 root data:ExchangeSender", + " └─ExchangeSender 6929.18 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 6929.18 mpp[tiflash] test.t.a, test.t.b", + " └─HashJoin 6929.18 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)], other cond:or(lt(test.t.a, 18), gt(test.t.b, 1))", + " ├─ExchangeReceiver(Build) 5543.34 mpp[tiflash] ", + " │ └─ExchangeSender 5543.34 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 5543.34 mpp[tiflash] not(isnull(test.t.a)), or(lt(test.t.a, 18), gt(test.t.b, 1))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "WITH CTE AS (SELECT /*+ MERGE(), MPP_1PHASE_AGG() */ count(*) as a, b FROM t WHERE t.a < 60 group by b) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "Plan": [ + "TableReader 3013.16 root data:ExchangeSender", + "└─ExchangeSender 3013.16 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 3013.16 mpp[tiflash] Column#20, Column#21", + " └─HashAgg 3013.16 mpp[tiflash] group by:Column#20, Column#21, funcs:firstrow(Column#20)->Column#20, funcs:firstrow(Column#21)->Column#21", + " └─ExchangeReceiver 3013.16 mpp[tiflash] ", + " └─ExchangeSender 3013.16 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: Column#20, collate: binary], [name: Column#21, collate: binary]", + " └─Union 3013.16 mpp[tiflash] ", + " ├─Projection 2126.93 mpp[tiflash] cast(Column#12, bigint(21) BINARY)->Column#20, test.t.b", + " │ └─Selection 2126.93 mpp[tiflash] lt(Column#12, 18)", + " │ └─Projection 2658.67 mpp[tiflash] Column#12, test.t.b", + " │ └─HashAgg 2658.67 mpp[tiflash] group by:test.t.b, funcs:count(1)->Column#12, funcs:firstrow(test.t.b)->test.t.b", + " │ └─ExchangeReceiver 3323.33 mpp[tiflash] ", + " │ └─ExchangeSender 3323.33 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " │ └─Selection 3323.33 mpp[tiflash] lt(test.t.a, 60)", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", + " └─Projection 886.22 mpp[tiflash] cast(Column#20, bigint(21) BINARY)->Column#20, Column#21", + " └─Projection 886.22 mpp[tiflash] Column#19, test.t.b", + " └─HashAgg 886.22 mpp[tiflash] group by:test.t.b, funcs:count(1)->Column#19, funcs:firstrow(test.t.b)->test.t.b", + " └─ExchangeReceiver 1107.78 mpp[tiflash] ", + " └─ExchangeSender 1107.78 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " └─Selection 1107.78 mpp[tiflash] gt(test.t.b, 1), lt(test.t.a, 60)", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "WITH CTE AS (SELECT /*+ MERGE(), MPP_2PHASE_AGG() */ count(*) as a, b FROM t WHERE t.a < 60 group by b) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "Plan": [ + "TableReader 3013.16 root data:ExchangeSender", + "└─ExchangeSender 3013.16 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 3013.16 mpp[tiflash] Column#20, Column#21", + " └─HashAgg 3013.16 mpp[tiflash] group by:Column#20, Column#21, funcs:firstrow(Column#20)->Column#20, funcs:firstrow(Column#21)->Column#21", + " └─ExchangeReceiver 3013.16 mpp[tiflash] ", + " └─ExchangeSender 3013.16 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: Column#20, collate: binary], [name: Column#21, collate: binary]", + " └─Union 3013.16 mpp[tiflash] ", + " ├─Projection 2126.93 mpp[tiflash] cast(Column#12, bigint(21) BINARY)->Column#20, test.t.b", + " │ └─Selection 2126.93 mpp[tiflash] lt(Column#12, 18)", + " │ └─Projection 2658.67 mpp[tiflash] Column#12, test.t.b", + " │ └─HashAgg 2658.67 mpp[tiflash] group by:test.t.b, funcs:sum(Column#32)->Column#12, funcs:firstrow(test.t.b)->test.t.b", + " │ └─ExchangeReceiver 2658.67 mpp[tiflash] ", + " │ └─ExchangeSender 2658.67 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " │ └─HashAgg 2658.67 mpp[tiflash] group by:test.t.b, funcs:count(1)->Column#32", + " │ └─Selection 3323.33 mpp[tiflash] lt(test.t.a, 60)", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", + " └─Projection 886.22 mpp[tiflash] cast(Column#20, bigint(21) BINARY)->Column#20, Column#21", + " └─Projection 886.22 mpp[tiflash] Column#19, test.t.b", + " └─HashAgg 886.22 mpp[tiflash] group by:test.t.b, funcs:sum(Column#46)->Column#19, funcs:firstrow(test.t.b)->test.t.b", + " └─ExchangeReceiver 886.22 mpp[tiflash] ", + " └─ExchangeSender 886.22 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.b, collate: binary]", + " └─HashAgg 886.22 mpp[tiflash] group by:test.t.b, funcs:count(1)->Column#46", + " └─Selection 1107.78 mpp[tiflash] gt(test.t.b, 1), lt(test.t.a, 60)", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "WITH CTE AS (SELECT /*+ MERGE(), shuffle_join(t1, t) */ t.a, t.b FROM t join t t1 where t.a = t1.a) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "Plan": [ + "TableReader 5322.67 root data:ExchangeSender", + "└─ExchangeSender 5322.67 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 5322.67 mpp[tiflash] Column#29, Column#30", + " └─HashAgg 5322.67 mpp[tiflash] group by:Column#29, Column#30, funcs:firstrow(Column#29)->Column#29, funcs:firstrow(Column#30)->Column#30", + " └─ExchangeReceiver 5322.67 mpp[tiflash] ", + " └─ExchangeSender 5322.67 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: Column#29, collate: binary], [name: Column#30, collate: binary]", + " └─HashAgg 5322.67 mpp[tiflash] group by:Column#29, Column#30, ", + " └─Union 8316.67 mpp[tiflash] ", + " ├─Projection 4154.17 mpp[tiflash] test.t.a, test.t.b", + " │ └─HashJoin 4154.17 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " │ ├─ExchangeReceiver(Build) 3323.33 mpp[tiflash] ", + " │ │ └─ExchangeSender 3323.33 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " │ │ └─Selection 3323.33 mpp[tiflash] lt(test.t.a, 18), not(isnull(test.t.a))", + " │ │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", + " │ └─ExchangeReceiver(Probe) 3323.33 mpp[tiflash] ", + " │ └─ExchangeSender 3323.33 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " │ └─Selection 3323.33 mpp[tiflash] lt(test.t.a, 18), not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Projection 4162.50 mpp[tiflash] test.t.a, test.t.b", + " └─HashJoin 4162.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 3330.00 mpp[tiflash] ", + " │ └─ExchangeSender 3330.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " │ └─Selection 3330.00 mpp[tiflash] gt(test.t.b, 1), not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 9990.00 mpp[tiflash] ", + " └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.a, collate: binary]", + " └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]There are no matching table names for (t1, t) in optimizer hint /*+ SHUFFLE_JOIN(t1, t, t1, t) */ or /*+ SHUFFLE_JOIN(t1, t, t1, t) */. Maybe you can use the table alias name", + "[planner:1815]There are no matching table names for (t1, t, t1, t) in optimizer hint /*+ SHUFFLE_JOIN(t1, t, t1, t, t1, t) */ or /*+ SHUFFLE_JOIN(t1, t, t1, t, t1, t) */. Maybe you can use the table alias name" + ] + }, + { + "SQL": "WITH CTE AS (SELECT /*+ MERGE(), broadcast_join(t1, t) */ t.a, t.b FROM t join t t1 where t.a = t1.a) SELECT * FROM CTE WHERE CTE.a <18 union select * from cte where cte.b > 1;", + "Plan": [ + "TableReader 5322.67 root data:ExchangeSender", + "└─ExchangeSender 5322.67 mpp[tiflash] ExchangeType: PassThrough", + " └─Projection 5322.67 mpp[tiflash] Column#29, Column#30", + " └─HashAgg 5322.67 mpp[tiflash] group by:Column#29, Column#30, funcs:firstrow(Column#29)->Column#29, funcs:firstrow(Column#30)->Column#30", + " └─ExchangeReceiver 5322.67 mpp[tiflash] ", + " └─ExchangeSender 5322.67 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: Column#29, collate: binary], [name: Column#30, collate: binary]", + " └─HashAgg 5322.67 mpp[tiflash] group by:Column#29, Column#30, ", + " └─Union 8316.67 mpp[tiflash] ", + " ├─Projection 4154.17 mpp[tiflash] test.t.a, test.t.b", + " │ └─HashJoin 4154.17 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " │ ├─ExchangeReceiver(Build) 3323.33 mpp[tiflash] ", + " │ │ └─ExchangeSender 3323.33 mpp[tiflash] ExchangeType: Broadcast", + " │ │ └─Selection 3323.33 mpp[tiflash] lt(test.t.a, 18), not(isnull(test.t.a))", + " │ │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", + " │ └─Selection(Probe) 3323.33 mpp[tiflash] lt(test.t.a, 18), not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Projection 4162.50 mpp[tiflash] test.t.a, test.t.b", + " └─HashJoin 4162.50 mpp[tiflash] inner join, equal:[eq(test.t.a, test.t.a)]", + " ├─ExchangeReceiver(Build) 3330.00 mpp[tiflash] ", + " │ └─ExchangeSender 3330.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 3330.00 mpp[tiflash] gt(test.t.b, 1), not(isnull(test.t.a))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.a))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo" + ], + "Warn": [ + "[planner:1815]There are no matching table names for (t1, t) in optimizer hint /*+ BROADCAST_JOIN(t1, t, t1, t) */ or /*+ TIDB_BCJ(t1, t, t1, t) */. Maybe you can use the table alias name", + "[planner:1815]There are no matching table names for (t1, t, t1, t) in optimizer hint /*+ BROADCAST_JOIN(t1, t, t1, t, t1, t) */ or /*+ TIDB_BCJ(t1, t, t1, t, t1, t) */. Maybe you can use the table alias name" ] } ] diff --git a/planner/optimize.go b/planner/optimize.go index 3a6804d5fa319..5e572d8485368 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -74,18 +74,25 @@ func matchSQLBinding(sctx sessionctx.Context, stmtNode ast.StmtNode) (bindRecord } // getPlanFromNonPreparedPlanCache tries to get an available cached plan from the NonPrepared Plan Cache for this stmt. -func getPlanFromNonPreparedPlanCache(ctx context.Context, sctx sessionctx.Context, stmt ast.StmtNode, is infoschema.InfoSchema) (core.Plan, types.NameSlice, bool, error) { +func getPlanFromNonPreparedPlanCache(ctx context.Context, sctx sessionctx.Context, stmt ast.StmtNode, is infoschema.InfoSchema) (p core.Plan, ns types.NameSlice, ok bool, err error) { if sctx.GetSessionVars().StmtCtx.InPreparedPlanBuilding || // already in cached plan rebuilding phase !core.NonPreparedPlanCacheableWithCtx(sctx, stmt, is) { return nil, nil, false, nil } - paramSQL, params, err := core.ParameterizeAST(sctx, stmt) + paramSQL, params, err := core.ParameterizeAST(ctx, sctx, stmt) if err != nil { return nil, nil, false, err } + defer func() { + if err != nil { + // keep the stmt unchanged if err so that it can fallback to the normal optimization path. + // TODO: add metrics + err = core.RestoreASTWithParams(ctx, sctx, stmt, params) + } + }() val := sctx.GetSessionVars().GetNonPreparedPlanCacheStmt(paramSQL) if val == nil { - cachedStmt, _, _, err := core.GeneratePlanCacheStmtWithAST(ctx, sctx, stmt) + cachedStmt, _, _, err := core.GeneratePlanCacheStmtWithAST(ctx, sctx, paramSQL, stmt) if err != nil { return nil, nil, false, err } @@ -234,6 +241,8 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in sessVars.FoundInBinding = true if sessVars.StmtCtx.InVerboseExplain { sessVars.StmtCtx.AppendNote(errors.Errorf("Using the bindSQL: %v", chosenBinding.BindSQL)) + } else { + sessVars.StmtCtx.AppendExtraNote(errors.Errorf("Using the bindSQL: %v", chosenBinding.BindSQL)) } } // Restore the hint to avoid changing the stmt node. diff --git a/server/BUILD.bazel b/server/BUILD.bazel index 191ae1c1c6f7d..afcb983fad76e 100644 --- a/server/BUILD.bazel +++ b/server/BUILD.bazel @@ -58,6 +58,7 @@ go_library( "//sessionctx/stmtctx", "//sessionctx/variable", "//sessiontxn", + "//statistics/handle", "//store", "//store/driver/error", "//store/gcworker", @@ -91,6 +92,7 @@ go_library( "//util/topsql/stmtstats", "//util/versioninfo", "@com_github_blacktear23_go_proxyprotocol//:go-proxyprotocol", + "@com_github_burntsushi_toml//:toml", "@com_github_gorilla_mux//:mux", "@com_github_opentracing_opentracing_go//:opentracing-go", "@com_github_pingcap_errors//:errors", diff --git a/server/conn_stmt.go b/server/conn_stmt.go index 43a2253fff477..acb12ae660b94 100644 --- a/server/conn_stmt.go +++ b/server/conn_stmt.go @@ -158,7 +158,10 @@ func (cc *clientConn) handleStmtExecute(ctx context.Context, data []byte) (err e return mysql.NewErrf(mysql.ErrUnknown, "unsupported flag: CursorTypeScrollable", nil) } - if !useCursor { + if useCursor { + cc.ctx.GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true) + defer cc.ctx.GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, false) + } else { // not using streaming ,can reuse chunk cc.ctx.GetSessionVars().SetAlloc(cc.chunkAlloc) } @@ -251,7 +254,8 @@ func (cc *clientConn) executePlanCacheStmt(ctx context.Context, stmt interface{} // The first return value indicates whether the call of executePreparedStmtAndWriteResult has no side effect and can be retried. // Currently the first return value is used to fallback to TiKV when TiFlash is down. func (cc *clientConn) executePreparedStmtAndWriteResult(ctx context.Context, stmt PreparedStatement, args []expression.Expression, useCursor bool) (bool, error) { - prepStmt, err := (&cc.ctx).GetSessionVars().GetPreparedStmtByID(uint32(stmt.ID())) + vars := (&cc.ctx).GetSessionVars() + prepStmt, err := vars.GetPreparedStmtByID(uint32(stmt.ID())) if err != nil { return true, errors.Annotate(err, cc.preparedStmt2String(uint32(stmt.ID()))) } @@ -264,6 +268,9 @@ func (cc *clientConn) executePreparedStmtAndWriteResult(ctx context.Context, stm return true, errors.Annotate(err, cc.preparedStmt2String(uint32(stmt.ID()))) } if rs == nil { + if useCursor { + vars.SetStatusFlag(mysql.ServerStatusCursorExists, false) + } return false, cc.writeOK(ctx) } // since there are multiple implementations of ResultSet (the rs might be wrapped), we have to unwrap the rs before @@ -294,7 +301,7 @@ func (cc *clientConn) executePreparedStmtAndWriteResult(ctx context.Context, stm cl.OnFetchReturned() } // explicitly flush columnInfo to client. - err = cc.writeEOF(ctx, cc.ctx.Status()|mysql.ServerStatusCursorExists) + err = cc.writeEOF(ctx, cc.ctx.Status()) if err != nil { return false, err } @@ -316,6 +323,8 @@ const ( func (cc *clientConn) handleStmtFetch(ctx context.Context, data []byte) (err error) { cc.ctx.GetSessionVars().StartTime = time.Now() cc.ctx.GetSessionVars().ClearAlloc(nil, false) + cc.ctx.GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true) + defer cc.ctx.GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, false) stmtID, fetchSize, err := parseStmtFetchCmd(data) if err != nil { @@ -344,7 +353,7 @@ func (cc *clientConn) handleStmtFetch(ctx context.Context, data []byte) (err err strconv.FormatUint(uint64(stmtID), 10), "stmt_fetch_rs"), cc.preparedStmt2String(stmtID)) } - _, err = cc.writeResultset(ctx, rs, true, cc.ctx.Status()|mysql.ServerStatusCursorExists, int(fetchSize)) + _, err = cc.writeResultset(ctx, rs, true, cc.ctx.Status(), int(fetchSize)) if err != nil { return errors.Annotate(err, cc.preparedStmt2String(stmtID)) } diff --git a/server/conn_stmt_test.go b/server/conn_stmt_test.go index 2e60fc1085332..366e5c54ac222 100644 --- a/server/conn_stmt_test.go +++ b/server/conn_stmt_test.go @@ -15,6 +15,7 @@ package server import ( + "bytes" "context" "encoding/binary" "testing" @@ -340,3 +341,59 @@ func TestCursorReadHoldTS(t *testing.T) { require.Zero(t, tk.Session().ShowProcess().GetMinStartTS(0)) require.Zero(t, srv.GetMinStartTS(0)) } + +func TestCursorExistsFlag(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + srv := CreateMockServer(t, store) + srv.SetDomain(dom) + defer srv.Close() + + appendUint32 := binary.LittleEndian.AppendUint32 + ctx := context.Background() + c := CreateMockConn(t, srv).(*mockConn) + out := new(bytes.Buffer) + c.pkt.bufWriter.Reset(out) + c.capability |= mysql.ClientDeprecateEOF | mysql.ClientProtocol41 + tk := testkit.NewTestKitWithSession(t, store, c.Context().Session) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int primary key)") + tk.MustExec("insert into t values (1), (2), (3), (4), (5), (6), (7), (8)") + tk.MustQuery("select count(*) from t").Check(testkit.Rows("8")) + + getLastStatus := func() uint16 { + raw := out.Bytes() + return binary.LittleEndian.Uint16(raw[len(raw)-4 : len(raw)-2]) + } + + stmt, _, _, err := c.Context().Prepare("select * from t") + require.NoError(t, err) + + require.NoError(t, c.Dispatch(ctx, append( + appendUint32([]byte{mysql.ComStmtExecute}, uint32(stmt.ID())), + mysql.CursorTypeReadOnly, 0x1, 0x0, 0x0, 0x0, + ))) + require.True(t, mysql.HasCursorExistsFlag(getLastStatus())) + + // fetch first 5 + require.NoError(t, c.Dispatch(ctx, appendUint32(appendUint32([]byte{mysql.ComStmtFetch}, uint32(stmt.ID())), 5))) + require.True(t, mysql.HasCursorExistsFlag(getLastStatus())) + + // COM_QUERY during fetch + require.NoError(t, c.Dispatch(ctx, append([]byte{mysql.ComQuery}, "select * from t"...))) + require.False(t, mysql.HasCursorExistsFlag(getLastStatus())) + + // fetch last 3 + require.NoError(t, c.Dispatch(ctx, appendUint32(appendUint32([]byte{mysql.ComStmtFetch}, uint32(stmt.ID())), 5))) + require.True(t, mysql.HasCursorExistsFlag(getLastStatus())) + + // final fetch with no row retured + // (tidb doesn't unset cursor-exists flag in the previous response like mysql, one more fetch is needed) + require.NoError(t, c.Dispatch(ctx, appendUint32(appendUint32([]byte{mysql.ComStmtFetch}, uint32(stmt.ID())), 5))) + require.False(t, mysql.HasCursorExistsFlag(getLastStatus())) + require.True(t, getLastStatus()&mysql.ServerStatusLastRowSend > 0) + + // COM_QUERY after fetch + require.NoError(t, c.Dispatch(ctx, append([]byte{mysql.ComQuery}, "select * from t"...))) + require.False(t, mysql.HasCursorExistsFlag(getLastStatus())) +} diff --git a/server/plan_replayer.go b/server/plan_replayer.go index 13160d3b6202f..beb202638d1fd 100644 --- a/server/plan_replayer.go +++ b/server/plan_replayer.go @@ -15,15 +15,26 @@ package server import ( + "archive/zip" + "bytes" + "encoding/json" "fmt" "io" "net/http" "os" "path/filepath" + "strconv" + "strings" + "github.com/BurntSushi/toml" "github.com/gorilla/mux" + "github.com/pingcap/errors" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/domain/infosync" + "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/parser/model" + "github.com/pingcap/tidb/statistics/handle" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/replayer" @@ -32,9 +43,11 @@ import ( // PlanReplayerHandler is the handler for dumping plan replayer file. type PlanReplayerHandler struct { - infoGetter *infosync.InfoSyncer - address string - statusPort uint + is infoschema.InfoSchema + statsHandle *handle.Handle + infoGetter *infosync.InfoSyncer + address string + statusPort uint } func (s *Server) newPlanReplayerHandler() *PlanReplayerHandler { @@ -46,6 +59,12 @@ func (s *Server) newPlanReplayerHandler() *PlanReplayerHandler { if s.dom != nil && s.dom.InfoSyncer() != nil { prh.infoGetter = s.dom.InfoSyncer() } + if s.dom != nil && s.dom.InfoSchema() != nil { + prh.is = s.dom.InfoSchema() + } + if s.dom != nil && s.dom.StatsHandle() != nil { + prh.statsHandle = s.dom.StatsHandle() + } return prh } @@ -61,6 +80,8 @@ func (prh PlanReplayerHandler) ServeHTTP(w http.ResponseWriter, req *http.Reques urlPath: fmt.Sprintf("plan_replayer/dump/%s", name), downloadedFilename: "plan_replayer", scheme: util.InternalHTTPSchema(), + statsHandle: prh.statsHandle, + is: prh.is, } handleDownloadFile(handler, w, req) } @@ -93,6 +114,13 @@ func handleDownloadFile(handler downloadFileHandler, w http.ResponseWriter, req writeError(w, err) return } + if handler.downloadedFilename == "plan_replayer" { + content, err = handlePlanReplayerContinuesCaptureFile(content, path, handler) + if err != nil { + writeError(w, err) + return + } + } _, err = w.Write(content) if err != nil { writeError(w, err) @@ -175,6 +203,9 @@ type downloadFileHandler struct { statusPort uint urlPath string downloadedFilename string + + statsHandle *handle.Handle + is infoschema.InfoSchema } func isExists(path string) (bool, error) { @@ -187,3 +218,162 @@ func isExists(path string) (bool, error) { } return true, nil } + +func handlePlanReplayerContinuesCaptureFile(content []byte, path string, handler downloadFileHandler) ([]byte, error) { + if !strings.Contains(handler.filePath, "continues_replayer") { + return content, nil + } + b := bytes.NewReader(content) + zr, err := zip.NewReader(b, int64(len(content))) + if err != nil { + return nil, err + } + startTS, err := loadSQLMetaFile(zr) + if err != nil { + return nil, err + } + if startTS == 0 { + return content, nil + } + tbls, err := loadSchemaMeta(zr, handler.is) + if err != nil { + return nil, err + } + for _, tbl := range tbls { + jsonStats, err := handler.statsHandle.DumpHistoricalStatsBySnapshot(tbl.dbName, tbl.info, startTS) + if err != nil { + return nil, err + } + tbl.jsonStats = jsonStats + } + newPath, err := dumpJSONStatsIntoZip(tbls, content, path) + if err != nil { + return nil, err + } + //nolint: gosec + file, err := os.Open(newPath) + if err != nil { + return nil, err + } + content, err = io.ReadAll(file) + if err != nil { + return nil, err + } + err = file.Close() + if err != nil { + return nil, err + } + return content, nil +} + +func loadSQLMetaFile(z *zip.Reader) (uint64, error) { + for _, zipFile := range z.File { + if zipFile.Name == domain.PlanReplayerSQLMetaFile { + varMap := make(map[string]string) + v, err := zipFile.Open() + if err != nil { + return 0, errors.AddStack(err) + } + //nolint: errcheck,all_revive + defer v.Close() + _, err = toml.DecodeReader(v, &varMap) + if err != nil { + return 0, errors.AddStack(err) + } + startTS, err := strconv.ParseUint(varMap[domain.PlanReplayerSQLMetaStartTS], 10, 64) + if err != nil { + return 0, err + } + return startTS, nil + } + } + return 0, nil +} + +func loadSchemaMeta(z *zip.Reader, is infoschema.InfoSchema) (map[int64]*tblInfo, error) { + r := make(map[int64]*tblInfo, 0) + for _, zipFile := range z.File { + if zipFile.Name == fmt.Sprintf("schema/%v", domain.PlanReplayerSchemaMetaFile) { + v, err := zipFile.Open() + if err != nil { + return nil, errors.AddStack(err) + } + //nolint: errcheck,all_revive + defer v.Close() + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(v) + if err != nil { + return nil, errors.AddStack(err) + } + rows := strings.Split(buf.String(), "\n") + for _, row := range rows { + s := strings.Split(row, ";") + databaseName := s[0] + tableName := s[1] + t, err := is.TableByName(model.NewCIStr(databaseName), model.NewCIStr(tableName)) + if err != nil { + return nil, err + } + r[t.Meta().ID] = &tblInfo{ + info: t.Meta(), + dbName: databaseName, + tblName: tableName, + } + } + break + } + } + return r, nil +} + +func dumpJSONStatsIntoZip(tbls map[int64]*tblInfo, content []byte, path string) (string, error) { + zr, err := zip.NewReader(bytes.NewReader(content), int64(len(content))) + if err != nil { + return "", err + } + newPath := fmt.Sprintf("copy_%v.zip", path[0:len(path)-4]) + zf, err := os.Create(newPath) + if err != nil { + return "", err + } + zw := zip.NewWriter(zf) + for _, f := range zr.File { + err = zw.Copy(f) + if err != nil { + logutil.BgLogger().Error("copy plan replayer zip file failed", zap.Error(err)) + return "", err + } + } + for _, tbl := range tbls { + w, err := zw.Create(fmt.Sprintf("stats/%v.%v.json", tbl.dbName, tbl.tblName)) + if err != nil { + return "", err + } + data, err := json.Marshal(tbl.jsonStats) + if err != nil { + return "", err + } + _, err = w.Write(data) + if err != nil { + return "", err + } + } + err = zw.Close() + if err != nil { + logutil.BgLogger().Error("Closing file failed", zap.Error(err)) + return "", err + } + err = zf.Close() + if err != nil { + logutil.BgLogger().Error("Closing file failed", zap.Error(err)) + return "", err + } + return newPath, nil +} + +type tblInfo struct { + info *model.TableInfo + jsonStats *handle.JSONTable + dbName string + tblName string +} diff --git a/server/server.go b/server/server.go index 09a20c8cb39c2..ba915c64f23cb 100644 --- a/server/server.go +++ b/server/server.go @@ -733,6 +733,11 @@ func (s *Server) GetProcessInfo(id uint64) (*util.ProcessInfo, bool) { conn, ok := s.clients[id] s.rwlock.RUnlock() if !ok { + if s.dom != nil { + if pinfo, ok2 := s.dom.SysProcTracker().GetSysProcessList()[id]; ok2 { + return pinfo, true + } + } return &util.ProcessInfo{}, false } return conn.ctx.ShowProcess(), ok diff --git a/session/session.go b/session/session.go index 2c6aa0567fa66..d358d761560e2 100644 --- a/session/session.go +++ b/session/session.go @@ -1988,6 +1988,7 @@ func (s *session) useCurrentSession(execOption sqlexec.ExecOption) (*session, fu s.sessionVars.StmtCtx.OriginalSQL = prevSQL s.sessionVars.StmtCtx.StmtType = prevStmtType s.sessionVars.StmtCtx.Tables = prevTables + s.sessionVars.MemTracker.Detach() }, nil } @@ -2049,6 +2050,7 @@ func (s *session) getInternalSession(execOption sqlexec.ExecOption) (*session, f se.sessionVars.PartitionPruneMode.Store(prePruneMode) se.sessionVars.OptimizerUseInvisibleIndexes = false se.sessionVars.InspectionTableCache = nil + se.sessionVars.MemTracker.Detach() s.sysSessionPool().Put(tmp) }, nil } @@ -2089,7 +2091,7 @@ func (s *session) ExecRestrictedSQL(ctx context.Context, opts []sqlexec.OptionFu metrics.SessionRestrictedSQLCounter.Inc() ctx = context.WithValue(ctx, execdetails.StmtExecDetailKey, &execdetails.StmtExecDetails{}) ctx = context.WithValue(ctx, tikvutil.ExecDetailsKey, &tikvutil.ExecDetails{}) - rs, err := se.ExecuteStmt(ctx, stmt) + rs, err := se.ExecuteInternalStmt(ctx, stmt) if err != nil { se.sessionVars.StmtCtx.AppendError(err) } @@ -2111,6 +2113,18 @@ func (s *session) ExecRestrictedSQL(ctx context.Context, opts []sqlexec.OptionFu }) } +// ExecuteInternalStmt execute internal stmt +func (s *session) ExecuteInternalStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlexec.RecordSet, error) { + origin := s.sessionVars.InRestrictedSQL + s.sessionVars.InRestrictedSQL = true + defer func() { + s.sessionVars.InRestrictedSQL = origin + // Restore the goroutine label by using the original ctx after execution is finished. + pprof.SetGoroutineLabels(ctx) + }() + return s.ExecuteStmt(ctx, stmtNode) +} + func (s *session) ExecuteStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlexec.RecordSet, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("session.ExecuteStmt", opentracing.ChildOf(span.Context())) @@ -4141,23 +4155,26 @@ func (s *session) setRequestSource(ctx context.Context, stmtLabel string, stmtNo } else { s.sessionVars.RequestSourceType = stmtLabel } - } else { - if source := ctx.Value(kv.RequestSourceKey); source != nil { - s.sessionVars.RequestSourceType = source.(kv.RequestSource).RequestSourceType - } else { - // panic in test mode in case there are requests without source in the future. - // log warnings in production mode. - if flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil { - panic("unexpected no source type context, if you see this error, " + - "the `RequestSourceTypeKey` is missing in your context") - } else { - logutil.Logger(ctx).Warn("unexpected no source type context, if you see this warning, "+ - "the `RequestSourceTypeKey` is missing in the context", - zap.Bool("internal", s.isInternal()), - zap.String("sql", stmtNode.Text())) - } + return + } + if source := ctx.Value(kv.RequestSourceKey); source != nil { + requestSource := source.(kv.RequestSource) + if requestSource.RequestSourceType != "" { + s.sessionVars.RequestSourceType = requestSource.RequestSourceType + return } } + // panic in test mode in case there are requests without source in the future. + // log warnings in production mode. + if flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil { + panic("unexpected no source type context, if you see this error, " + + "the `RequestSourceTypeKey` is missing in your context") + } else { + logutil.Logger(ctx).Warn("unexpected no source type context, if you see this warning, "+ + "the `RequestSourceTypeKey` is missing in the context", + zap.Bool("internal", s.isInternal()), + zap.String("sql", stmtNode.Text())) + } } // RemoveLockDDLJobs removes the DDL jobs which doesn't get the metadata lock from job2ver. diff --git a/sessionctx/sessionstates/session_states.go b/sessionctx/sessionstates/session_states.go index 36ea0b22455d7..c9e1652a9c1df 100644 --- a/sessionctx/sessionstates/session_states.go +++ b/sessionctx/sessionstates/session_states.go @@ -15,8 +15,6 @@ package sessionstates import ( - "time" - "github.com/pingcap/tidb/errno" ptypes "github.com/pingcap/tidb/parser/types" "github.com/pingcap/tidb/sessionctx/stmtctx" @@ -79,7 +77,6 @@ type SessionStates struct { FoundInPlanCache bool `json:"in-plan-cache,omitempty"` FoundInBinding bool `json:"in-binding,omitempty"` SequenceLatestValues map[int64]int64 `json:"seq-values,omitempty"` - MPPStoreLastFailTime map[string]time.Time `json:"store-fail-time,omitempty"` LastAffectedRows int64 `json:"affected-rows,omitempty"` LastInsertID uint64 `json:"last-insert-id,omitempty"` Warnings []stmtctx.SQLWarn `json:"warnings,omitempty"` diff --git a/sessionctx/sessionstates/session_states_test.go b/sessionctx/sessionstates/session_states_test.go index 5910d6b18e071..21de8d53727d6 100644 --- a/sessionctx/sessionstates/session_states_test.go +++ b/sessionctx/sessionstates/session_states_test.go @@ -20,9 +20,7 @@ import ( "fmt" "strconv" "strings" - "sync" "testing" - "time" "github.com/pingcap/errors" "github.com/pingcap/tidb/config" @@ -133,7 +131,7 @@ func TestSystemVars(t *testing.T) { { // sem invisible variable inSessionStates: false, - varName: variable.TiDBAllowRemoveAutoInc, + varName: variable.TiDBConfig, }, { // noop variables @@ -378,23 +376,6 @@ func TestSessionCtx(t *testing.T) { tk.MustQuery("select nextval(test.s)").Check(testkit.Rows("2")) }, }, - { - // check MPPStoreLastFailTime - setFunc: func(tk *testkit.TestKit) any { - m := sync.Map{} - m.Store("store1", time.Now()) - tk.Session().GetSessionVars().MPPStoreLastFailTime = &m - return tk.Session().GetSessionVars().MPPStoreLastFailTime - }, - checkFunc: func(tk *testkit.TestKit, param any) { - failTime := tk.Session().GetSessionVars().MPPStoreLastFailTime - tm, ok := failTime.Load("store1") - require.True(t, ok) - v, ok := (param.(*sync.Map)).Load("store1") - require.True(t, ok) - require.True(t, tm.(time.Time).Equal(v.(time.Time))) - }, - }, { // check FoundInPlanCache setFunc: func(tk *testkit.TestKit) any { diff --git a/sessionctx/stmtctx/BUILD.bazel b/sessionctx/stmtctx/BUILD.bazel index b42939f7ec324..a2e12d1e860e1 100644 --- a/sessionctx/stmtctx/BUILD.bazel +++ b/sessionctx/stmtctx/BUILD.bazel @@ -6,6 +6,7 @@ go_library( importpath = "github.com/pingcap/tidb/sessionctx/stmtctx", visibility = ["//visibility:public"], deps = [ + "//errno", "//parser", "//parser/ast", "//parser/model", diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index 6f9a276691149..9a49c24851c08 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -24,6 +24,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/parser" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/model" @@ -209,8 +210,15 @@ type StatementContext struct { copied uint64 touched uint64 - message string - warnings []SQLWarn + message string + warnings []SQLWarn + // extraWarnings record the extra warnings and are only used by the slow log only now. + // If a warning is expected to be output only under some conditions (like in EXPLAIN or EXPLAIN VERBOSE) but it's + // not under such conditions now, it is considered as an extra warning. + // extraWarnings would not be printed through SHOW WARNINGS, but we want to always output them through the slow + // log to help diagnostics, so we store them here separately. + extraWarnings []SQLWarn + execDetails execdetails.ExecDetails allExecDetails []*execdetails.DetailsNeedP90 } @@ -299,8 +307,6 @@ type StatementContext struct { LogOnExceed [2]memory.LogOnExceed } - // OptimInfo maps Plan.ID() to optimization information when generating Plan. - OptimInfo map[int]string // InVerboseExplain indicates the statement is "explain format='verbose' ...". InVerboseExplain bool @@ -812,6 +818,47 @@ func (sc *StatementContext) AppendError(warn error) { } } +// GetExtraWarnings gets extra warnings. +func (sc *StatementContext) GetExtraWarnings() []SQLWarn { + sc.mu.Lock() + defer sc.mu.Unlock() + return sc.mu.extraWarnings +} + +// SetExtraWarnings sets extra warnings. +func (sc *StatementContext) SetExtraWarnings(warns []SQLWarn) { + sc.mu.Lock() + defer sc.mu.Unlock() + sc.mu.extraWarnings = warns +} + +// AppendExtraWarning appends an extra warning with level 'Warning'. +func (sc *StatementContext) AppendExtraWarning(warn error) { + sc.mu.Lock() + defer sc.mu.Unlock() + if len(sc.mu.extraWarnings) < math.MaxUint16 { + sc.mu.extraWarnings = append(sc.mu.extraWarnings, SQLWarn{WarnLevelWarning, warn}) + } +} + +// AppendExtraNote appends an extra warning with level 'Note'. +func (sc *StatementContext) AppendExtraNote(warn error) { + sc.mu.Lock() + defer sc.mu.Unlock() + if len(sc.mu.extraWarnings) < math.MaxUint16 { + sc.mu.extraWarnings = append(sc.mu.extraWarnings, SQLWarn{WarnLevelNote, warn}) + } +} + +// AppendExtraError appends an extra warning with level 'Error'. +func (sc *StatementContext) AppendExtraError(warn error) { + sc.mu.Lock() + defer sc.mu.Unlock() + if len(sc.mu.extraWarnings) < math.MaxUint16 { + sc.mu.extraWarnings = append(sc.mu.extraWarnings, SQLWarn{WarnLevelError, warn}) + } +} + // HandleTruncate ignores or returns the error based on the StatementContext state. func (sc *StatementContext) HandleTruncate(err error) error { // TODO: At present we have not checked whether the error can be ignored or treated as warning. @@ -819,6 +866,21 @@ func (sc *StatementContext) HandleTruncate(err error) error { if err == nil { return nil } + + err = errors.Cause(err) + if e, ok := err.(*errors.Error); !ok || + (e.Code() != errno.ErrTruncatedWrongValue && + e.Code() != errno.ErrDataTooLong && + e.Code() != errno.ErrTruncatedWrongValueForField && + e.Code() != errno.ErrWarnDataOutOfRange && + e.Code() != errno.ErrDataOutOfRange && + e.Code() != errno.ErrBadNumber && + e.Code() != errno.ErrWrongValueForType && + e.Code() != errno.ErrDatetimeFunctionOverflow && + e.Code() != errno.WarnDataTruncated) { + return err + } + if sc.IgnoreTruncate { return nil } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index fe4972fb5ff3a..d3d30d9c7bcc5 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1171,9 +1171,6 @@ type SessionVars struct { // TemporaryTableData stores committed kv values for temporary table for current session. TemporaryTableData TemporaryTableData - // MPPStoreLastFailTime records the lastest fail time that a TiFlash store failed. It maps store address(string) to fail time(time.Time). - MPPStoreLastFailTime *sync.Map - // MPPStoreFailTTL indicates the duration that protect TiDB from sending task to a new recovered TiFlash. MPPStoreFailTTL string @@ -1482,8 +1479,13 @@ func (s *SessionVars) IsMPPEnforced() bool { // TODO: Confirm whether this function will be inlined and // omit the overhead of string construction when calling with false condition. func (s *SessionVars) RaiseWarningWhenMPPEnforced(warning string) { - if s.IsMPPEnforced() && s.StmtCtx.InExplainStmt { + if !s.IsMPPEnforced() { + return + } + if s.StmtCtx.InExplainStmt { s.StmtCtx.AppendWarning(errors.New(warning)) + } else { + s.StmtCtx.AppendExtraWarning(errors.New(warning)) } } @@ -1708,7 +1710,6 @@ func NewSessionVars(hctx HookContext) *SessionVars { AllowFallbackToTiKV: make(map[kv.StoreType]struct{}), CTEMaxRecursionDepth: DefCTEMaxRecursionDepth, TMPTableSize: DefTiDBTmpTableMaxSize, - MPPStoreLastFailTime: new(sync.Map), MPPStoreFailTTL: DefTiDBMPPStoreFailTTL, Rng: mathutil.NewWithTime(), StatsLoadSyncWait: StatsLoadSyncWait.Load(), @@ -2331,12 +2332,6 @@ func (s *SessionVars) EncodeSessionStates(ctx context.Context, sessionStates *se } sessionStates.LastFoundRows = s.LastFoundRows sessionStates.SequenceLatestValues = s.SequenceState.GetAllStates() - sessionStates.MPPStoreLastFailTime = make(map[string]time.Time, 0) - s.MPPStoreLastFailTime.Range( - func(key, value interface{}) bool { - sessionStates.MPPStoreLastFailTime[key.(string)] = value.(time.Time) - return true - }) sessionStates.FoundInPlanCache = s.PrevFoundInPlanCache sessionStates.FoundInBinding = s.PrevFoundInBinding @@ -2372,9 +2367,6 @@ func (s *SessionVars) DecodeSessionStates(ctx context.Context, sessionStates *se } s.LastFoundRows = sessionStates.LastFoundRows s.SequenceState.SetAllStates(sessionStates.SequenceLatestValues) - for k, v := range sessionStates.MPPStoreLastFailTime { - s.MPPStoreLastFailTime.Store(k, v) - } s.FoundInPlanCache = sessionStates.FoundInPlanCache s.FoundInBinding = sessionStates.FoundInBinding diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 00137ca7608cf..c8da3ed5c10e6 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -1107,7 +1107,7 @@ const ( DefAdaptiveClosestReadThreshold = 4096 DefTiDBEnableAnalyzeSnapshot = false DefTiDBGenerateBinaryPlan = true - DefEnableTiDBGCAwareMemoryTrack = true + DefEnableTiDBGCAwareMemoryTrack = false DefTiDBDefaultStrMatchSelectivity = 0.8 DefTiDBEnableTmpStorageOnOOM = true DefTiDBEnableMDL = true diff --git a/sessiontxn/txn_rc_tso_optimize_test.go b/sessiontxn/txn_rc_tso_optimize_test.go index f321d40340496..3e945778ca107 100644 --- a/sessiontxn/txn_rc_tso_optimize_test.go +++ b/sessiontxn/txn_rc_tso_optimize_test.go @@ -105,7 +105,7 @@ func TestRcTSOCmdCountForPrepareExecuteNormal(t *testing.T) { tk.MustExec("commit") } countTsoRequest, countTsoUseConstant, countWaitTsoOracle := getAllTsoCounter(sctx) - require.Equal(t, uint64(398), countTsoRequest.(uint64)) + require.Equal(t, uint64(496), countTsoRequest.(uint64)) require.Equal(t, uint64(594), countTsoUseConstant.(uint64)) require.Equal(t, uint64(198), countWaitTsoOracle.(uint64)) @@ -137,7 +137,7 @@ func TestRcTSOCmdCountForPrepareExecuteNormal(t *testing.T) { tk.MustExec("commit") } count := sctx.Value(sessiontxn.TsoRequestCount) - require.Equal(t, uint64(594), count) + require.Equal(t, uint64(693), count) } func TestRcTSOCmdCountForPrepareExecuteExtra(t *testing.T) { @@ -234,7 +234,7 @@ func TestRcTSOCmdCountForPrepareExecuteExtra(t *testing.T) { tk.MustExec("commit") } countTsoRequest, countTsoUseConstant, countWaitTsoOracle = getAllTsoCounter(sctx) - require.Equal(t, uint64(16), countTsoRequest.(uint64)) + require.Equal(t, uint64(20), countTsoRequest.(uint64)) require.Equal(t, uint64(5), countTsoUseConstant.(uint64)) require.Equal(t, uint64(5), countWaitTsoOracle.(uint64)) @@ -412,7 +412,7 @@ func TestRcTSOCmdCountForPrepareExecuteExtra(t *testing.T) { require.Nil(t, stmt) tk.MustExec("commit") countTsoRequest, countTsoUseConstant, countWaitTsoOracle = getAllTsoCounter(sctx) - require.Equal(t, uint64(3), countTsoRequest.(uint64)) + require.Equal(t, uint64(4), countTsoRequest.(uint64)) require.Equal(t, uint64(2), countTsoUseConstant.(uint64)) require.Equal(t, 0, countWaitTsoOracle.(int)) tk.MustQuery("SELECT * FROM t1 WHERE id1 = 1").Check(testkit.Rows("1 1 1")) diff --git a/statistics/handle/gc.go b/statistics/handle/gc.go index ab2159bf22bf0..46a2a9fc691e3 100644 --- a/statistics/handle/gc.go +++ b/statistics/handle/gc.go @@ -160,6 +160,7 @@ func (h *Handle) ClearOutdatedHistoryStats() error { } sql = "delete from mysql.stats_history where NOW() - create_time >= %? " _, err = exec.ExecuteInternal(ctx, sql, variable.HistoricalStatsDuration.Load().Seconds()) + logutil.BgLogger().Info("clear outdated historical stats") return err } diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index c85656ff99d4b..f53f075301acb 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -1906,7 +1906,9 @@ func (h *Handle) histogramFromStorage(reader *statsReader, tableID int64, colID lowerBound = rows[i].GetDatum(2, &fields[2].Column.FieldType) upperBound = rows[i].GetDatum(3, &fields[3].Column.FieldType) } else { - sc := &stmtctx.StatementContext{TimeZone: time.UTC} + // Invalid date values may be inserted into table under some relaxed sql mode. Those values may exist in statistics. + // Hence, when reading statistics, we should skip invalid date check. See #39336. + sc := &stmtctx.StatementContext{TimeZone: time.UTC, AllowInvalidDate: true, IgnoreZeroInDate: true} d := rows[i].GetDatum(2, &fields[2].Column.FieldType) // For new collation data, when storing the bounds of the histogram, we store the collate key instead of the // original value. diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 5e41b1c1279a2..9bb80498bc90f 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -3543,3 +3543,35 @@ func TestStatsLockAndUnlockTables(t *testing.T) { tbl2Stats2 := handle.GetTableStats(tbl2.Meta()) require.Equal(t, int64(2), tbl2Stats2.Count) } + +func TestIssue39336(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec(` +create table t1 ( + a datetime(3) default null, + b int +) partition by range (b) ( + partition p0 values less than (1000), + partition p1 values less than (maxvalue) +)`) + tk.MustExec("set @@sql_mode=''") + tk.MustExec("set @@tidb_analyze_version=2") + tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") + tk.MustExec(` +insert into t1 values +('1000-00-09 00:00:00.000', 1), +('1000-00-06 00:00:00.000', 1), +('1000-00-06 00:00:00.000', 1), +('2022-11-23 14:24:30.000', 1), +('2022-11-23 14:24:32.000', 1), +('2022-11-23 14:24:33.000', 1), +('2022-11-23 14:24:35.000', 1), +('2022-11-23 14:25:08.000', 1001), +('2022-11-23 14:25:09.000', 1001)`) + tk.MustExec("analyze table t1 with 0 topn") + rows := tk.MustQuery("show analyze status where job_info like 'merge global stats%'").Rows() + require.Len(t, rows, 1) + require.Equal(t, "finished", rows[0][7]) +} diff --git a/store/copr/BUILD.bazel b/store/copr/BUILD.bazel index 409c2add6f1da..9ea8467d01dfa 100644 --- a/store/copr/BUILD.bazel +++ b/store/copr/BUILD.bazel @@ -9,6 +9,7 @@ go_library( "coprocessor_cache.go", "key_ranges.go", "mpp.go", + "mpp_probe.go", "region_cache.go", "store.go", ], @@ -65,6 +66,7 @@ go_test( "coprocessor_test.go", "key_ranges_test.go", "main_test.go", + "mpp_probe_test.go", ], embed = [":copr"], flaky = True, @@ -74,12 +76,15 @@ go_test( "//store/driver/backoff", "//testkit/testsetup", "//util/paging", + "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/coprocessor", + "@com_github_pingcap_kvproto//pkg/mpp", "@com_github_stathat_consistent//:consistent", "@com_github_stretchr_testify//require", "@com_github_tikv_client_go_v2//config", "@com_github_tikv_client_go_v2//testutils", "@com_github_tikv_client_go_v2//tikv", + "@com_github_tikv_client_go_v2//tikvrpc", "@org_uber_go_goleak//:goleak", ], ) diff --git a/store/copr/batch_coprocessor.go b/store/copr/batch_coprocessor.go index 2a746c24dcf0d..47445a7b63577 100644 --- a/store/copr/batch_coprocessor.go +++ b/store/copr/batch_coprocessor.go @@ -29,7 +29,6 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/coprocessor" "github.com/pingcap/kvproto/pkg/kvrpcpb" - "github.com/pingcap/kvproto/pkg/mpp" "github.com/pingcap/log" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/kv" @@ -294,12 +293,11 @@ func balanceBatchCopTaskWithContinuity(storeTaskMap map[uint64]*batchCopTask, ca // // The second balance strategy: Not only consider the region count between TiFlash stores, but also try to make the regions' range continuous(stored in TiFlash closely). // If balanceWithContinuity is true, the second balance strategy is enable. -func balanceBatchCopTask(ctx context.Context, kvStore *kvStore, originalTasks []*batchCopTask, mppStoreLastFailTime *sync.Map, ttl time.Duration, balanceWithContinuity bool, balanceContinuousRegionCount int64) []*batchCopTask { +func balanceBatchCopTask(ctx context.Context, kvStore *kvStore, originalTasks []*batchCopTask, isMPP bool, ttl time.Duration, balanceWithContinuity bool, balanceContinuousRegionCount int64) []*batchCopTask { if len(originalTasks) == 0 { log.Info("Batch cop task balancer got an empty task set.") return originalTasks } - isMPP := mppStoreLastFailTime != nil // for mpp, we still need to detect the store availability if len(originalTasks) <= 1 && !isMPP { return originalTasks @@ -325,13 +323,35 @@ func balanceBatchCopTask(ctx context.Context, kvStore *kvStore, originalTasks [] } } else { stores := cache.RegionCache.GetTiFlashStores() - aliveStores := filterAliveStores(ctx, stores, mppStoreLastFailTime, ttl, kvStore) - for _, s := range aliveStores { - storeTaskMap[s.StoreID()] = &batchCopTask{ - storeAddr: s.GetAddr(), - cmdType: originalTasks[0].cmdType, - ctx: &tikv.RPCContext{Addr: s.GetAddr(), Store: s}, - } + var wg sync.WaitGroup + var mu sync.Mutex + wg.Add(len(stores)) + for i := range stores { + go func(idx int) { + defer wg.Done() + s := stores[idx] + + // check if store is failed already. + ok := GlobalMPPFailedStoreProber.IsRecovery(ctx, s.GetAddr(), ttl) + if !ok { + return + } + + tikvClient := kvStore.GetTiKVClient() + ok = detectMPPStore(ctx, tikvClient, s.GetAddr(), DetectTimeoutLimit) + if !ok { + GlobalMPPFailedStoreProber.Add(ctx, s.GetAddr(), tikvClient) + return + } + + mu.Lock() + defer mu.Unlock() + storeTaskMap[s.StoreID()] = &batchCopTask{ + storeAddr: s.GetAddr(), + cmdType: originalTasks[0].cmdType, + ctx: &tikv.RPCContext{Addr: s.GetAddr(), Store: s}, + } + }(i) } } @@ -484,29 +504,29 @@ func buildBatchCopTasksForNonPartitionedTable(bo *backoff.Backoffer, store *kvStore, ranges *KeyRanges, storeType kv.StoreType, - mppStoreLastFailTime *sync.Map, + isMPP bool, ttl time.Duration, balanceWithContinuity bool, balanceContinuousRegionCount int64) ([]*batchCopTask, error) { if config.GetGlobalConfig().DisaggregatedTiFlash { - return buildBatchCopTasksConsistentHash(bo, store, []*KeyRanges{ranges}, storeType, mppStoreLastFailTime, ttl) + return buildBatchCopTasksConsistentHash(bo, store, []*KeyRanges{ranges}, storeType, isMpp, ttl) } - return buildBatchCopTasksCore(bo, store, []*KeyRanges{ranges}, storeType, mppStoreLastFailTime, ttl, balanceWithContinuity, balanceContinuousRegionCount) + return buildBatchCopTasksCore(bo, store, []*KeyRanges{ranges}, storeType, isMPP, ttl, balanceWithContinuity, balanceContinuousRegionCount) } func buildBatchCopTasksForPartitionedTable(bo *backoff.Backoffer, store *kvStore, rangesForEachPhysicalTable []*KeyRanges, storeType kv.StoreType, - mppStoreLastFailTime *sync.Map, + isMPP bool, ttl time.Duration, balanceWithContinuity bool, balanceContinuousRegionCount int64, partitionIDs []int64) (batchTasks []*batchCopTask, err error) { if config.GetGlobalConfig().DisaggregatedTiFlash { - batchTasks, err = buildBatchCopTasksConsistentHash(bo, store, rangesForEachPhysicalTable, storeType, mppStoreLastFailTime, ttl) + batchTasks, err = buildBatchCopTasksConsistentHash(bo, store, rangesForEachPhysicalTable, storeType, isMPP, ttl) } else { - batchTasks, err = buildBatchCopTasksCore(bo, store, rangesForEachPhysicalTable, storeType, mppStoreLastFailTime, ttl, balanceWithContinuity, balanceContinuousRegionCount) + batchTasks, err = buildBatchCopTasksCore(bo, store, rangesForEachPhysicalTable, storeType, isMPP, ttl, balanceWithContinuity, balanceContinuousRegionCount) } if err != nil { return nil, err @@ -669,7 +689,7 @@ func buildBatchCopTasksConsistentHash(bo *backoff.Backoffer, // When `partitionIDs != nil`, it means that buildBatchCopTasksCore is constructing a batch cop tasks for PartitionTableScan. // At this time, `len(rangesForEachPhysicalTable) == len(partitionIDs)` and `rangesForEachPhysicalTable[i]` is for partition `partitionIDs[i]`. // Otherwise, `rangesForEachPhysicalTable[0]` indicates the range for the single physical table. -func buildBatchCopTasksCore(bo *backoff.Backoffer, store *kvStore, rangesForEachPhysicalTable []*KeyRanges, storeType kv.StoreType, mppStoreLastFailTime *sync.Map, ttl time.Duration, balanceWithContinuity bool, balanceContinuousRegionCount int64) ([]*batchCopTask, error) { +func buildBatchCopTasksCore(bo *backoff.Backoffer, store *kvStore, rangesForEachPhysicalTable []*KeyRanges, storeType kv.StoreType, isMPP bool, ttl time.Duration, balanceWithContinuity bool, balanceContinuousRegionCount int64) ([]*batchCopTask, error) { cache := store.GetRegionCache() start := time.Now() const cmdType = tikvrpc.CmdBatchCop @@ -699,7 +719,6 @@ func buildBatchCopTasksCore(bo *backoff.Backoffer, store *kvStore, rangesForEach storeTaskMap := make(map[string]*batchCopTask) needRetry := false - isMPP := mppStoreLastFailTime != nil for _, task := range tasks { rpcCtx, err := cache.GetTiFlashRPCContext(bo.TiKVBackoffer(), task.region, isMPP) if err != nil { @@ -751,7 +770,7 @@ func buildBatchCopTasksCore(bo *backoff.Backoffer, store *kvStore, rangesForEach logutil.BgLogger().Debug(msg) } balanceStart := time.Now() - batchTasks = balanceBatchCopTask(bo.GetCtx(), store, batchTasks, mppStoreLastFailTime, ttl, balanceWithContinuity, balanceContinuousRegionCount) + batchTasks = balanceBatchCopTask(bo.GetCtx(), store, batchTasks, isMPP, ttl, balanceWithContinuity, balanceContinuousRegionCount) balanceElapsed := time.Since(balanceStart) if log.GetLevel() <= zap.DebugLevel { msg := "After region balance:" @@ -817,11 +836,11 @@ func (c *CopClient) sendBatch(ctx context.Context, req *kv.Request, vars *tikv.V keyRanges = append(keyRanges, NewKeyRanges(pi.KeyRanges)) partitionIDs = append(partitionIDs, pi.ID) } - tasks, err = buildBatchCopTasksForPartitionedTable(bo, c.store.kvStore, keyRanges, req.StoreType, nil, 0, false, 0, partitionIDs) + tasks, err = buildBatchCopTasksForPartitionedTable(bo, c.store.kvStore, keyRanges, req.StoreType, false, 0, false, 0, partitionIDs) } else { // TODO: merge the if branch. ranges := NewKeyRanges(req.KeyRanges.FirstPartitionRange()) - tasks, err = buildBatchCopTasksForNonPartitionedTable(bo, c.store.kvStore, ranges, req.StoreType, nil, 0, false, 0) + tasks, err = buildBatchCopTasksForNonPartitionedTable(bo, c.store.kvStore, ranges, req.StoreType, false, 0, false, 0) } if err != nil { @@ -968,7 +987,7 @@ func (b *batchCopIterator) retryBatchCopTask(ctx context.Context, bo *backoff.Ba ranges = append(ranges, *ran) }) } - ret, err := buildBatchCopTasksForNonPartitionedTable(bo, b.store, NewKeyRanges(ranges), b.req.StoreType, nil, 0, false, 0) + ret, err := buildBatchCopTasksForNonPartitionedTable(bo, b.store, NewKeyRanges(ranges), b.req.StoreType, false, 0, false, 0) return ret, err } // Retry Partition Table Scan @@ -987,7 +1006,7 @@ func (b *batchCopIterator) retryBatchCopTask(ctx context.Context, bo *backoff.Ba } keyRanges = append(keyRanges, NewKeyRanges(ranges)) } - ret, err := buildBatchCopTasksForPartitionedTable(bo, b.store, keyRanges, b.req.StoreType, nil, 0, false, 0, pid) + ret, err := buildBatchCopTasksForPartitionedTable(bo, b.store, keyRanges, b.req.StoreType, false, 0, false, 0, pid) return ret, err } diff --git a/store/copr/batch_coprocessor_test.go b/store/copr/batch_coprocessor_test.go index 5616f61c54365..3e10ce627b1f6 100644 --- a/store/copr/batch_coprocessor_test.go +++ b/store/copr/batch_coprocessor_test.go @@ -120,13 +120,13 @@ func TestBalanceBatchCopTaskWithContinuity(t *testing.T) { func TestBalanceBatchCopTaskWithEmptyTaskSet(t *testing.T) { { var nilTaskSet []*batchCopTask - nilResult := balanceBatchCopTask(nil, nil, nilTaskSet, nil, time.Second, false, 0) + nilResult := balanceBatchCopTask(nil, nil, nilTaskSet, false, time.Second, false, 0) require.True(t, nilResult == nil) } { emptyTaskSet := make([]*batchCopTask, 0) - emptyResult := balanceBatchCopTask(nil, nil, emptyTaskSet, nil, time.Second, false, 0) + emptyResult := balanceBatchCopTask(nil, nil, emptyTaskSet, false, time.Second, false, 0) require.True(t, emptyResult != nil) require.True(t, len(emptyResult) == 0) } diff --git a/store/copr/mpp.go b/store/copr/mpp.go index 7dad1ce6aa349..06071edb332e8 100644 --- a/store/copr/mpp.go +++ b/store/copr/mpp.go @@ -62,7 +62,7 @@ func (c *MPPClient) selectAllTiFlashStore() []kv.MPPTaskMeta { } // ConstructMPPTasks receives ScheduleRequest, which are actually collects of kv ranges. We allocates MPPTaskMeta for them and returns. -func (c *MPPClient) ConstructMPPTasks(ctx context.Context, req *kv.MPPBuildTasksRequest, mppStoreLastFailTime *sync.Map, ttl time.Duration) ([]kv.MPPTaskMeta, error) { +func (c *MPPClient) ConstructMPPTasks(ctx context.Context, req *kv.MPPBuildTasksRequest, ttl time.Duration) ([]kv.MPPTaskMeta, error) { ctx = context.WithValue(ctx, tikv.TxnStartKey(), req.StartTS) bo := backoff.NewBackofferWithVars(ctx, copBuildTaskMaxBackoff, nil) var tasks []*batchCopTask @@ -74,13 +74,13 @@ func (c *MPPClient) ConstructMPPTasks(ctx context.Context, req *kv.MPPBuildTasks rangesForEachPartition[i] = NewKeyRanges(p.KeyRanges) partitionIDs[i] = p.ID } - tasks, err = buildBatchCopTasksForPartitionedTable(bo, c.store, rangesForEachPartition, kv.TiFlash, mppStoreLastFailTime, ttl, true, 20, partitionIDs) + tasks, err = buildBatchCopTasksForPartitionedTable(bo, c.store, rangesForEachPartition, kv.TiFlash, true, ttl, true, 20, partitionIDs) } else { if req.KeyRanges == nil { return c.selectAllTiFlashStore(), nil } ranges := NewKeyRanges(req.KeyRanges) - tasks, err = buildBatchCopTasksForNonPartitionedTable(bo, c.store, ranges, kv.TiFlash, mppStoreLastFailTime, ttl, true, 20) + tasks, err = buildBatchCopTasksForNonPartitionedTable(bo, c.store, ranges, kv.TiFlash, true, ttl, true, 20) } if err != nil { diff --git a/store/copr/mpp_probe.go b/store/copr/mpp_probe.go new file mode 100644 index 0000000000000..0a0eba286648e --- /dev/null +++ b/store/copr/mpp_probe.go @@ -0,0 +1,270 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package copr + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/mpp" + "github.com/pingcap/tidb/metrics" + "github.com/pingcap/tidb/util/logutil" + "github.com/tikv/client-go/v2/tikv" + "github.com/tikv/client-go/v2/tikvrpc" + "go.uber.org/zap" +) + +// GlobalMPPFailedStoreProber mpp failed store probe +var GlobalMPPFailedStoreProber *MPPFailedStoreProber + +const ( + // DetectPeriod detect period + DetectPeriod = 3 * time.Second + // DetectTimeoutLimit detect timeout + DetectTimeoutLimit = 2 * time.Second + // MaxRecoveryTimeLimit wait TiFlash recovery,more than MPPStoreFailTTL + MaxRecoveryTimeLimit = 15 * time.Minute + // MaxObsoletTimeLimit no request for a long time,that might be obsoleted + MaxObsoletTimeLimit = time.Hour +) + +// MPPStoreState the state for MPPStore. +type MPPStoreState struct { + address string // MPPStore TiFlash address + tikvClient tikv.Client + + lock struct { + sync.Mutex + + recoveryTime time.Time + lastLookupTime time.Time + lastDetectTime time.Time + } +} + +// MPPFailedStoreProber use for detecting of failed TiFlash instance +type MPPFailedStoreProber struct { + failedMPPStores *sync.Map + lock *sync.Mutex + isStop *atomic.Bool + wg *sync.WaitGroup + ctx context.Context + cancel context.CancelFunc + + detectPeriod time.Duration + detectTimeoutLimit time.Duration + maxRecoveryTimeLimit time.Duration + maxObsoletTimeLimit time.Duration +} + +func (t *MPPStoreState) detect(ctx context.Context, detectPeriod time.Duration, detectTimeoutLimit time.Duration) { + if time.Since(t.lock.lastDetectTime) < detectPeriod { + return + } + + defer func() { t.lock.lastDetectTime = time.Now() }() + metrics.TiFlashFailedMPPStoreState.WithLabelValues(t.address).Set(0) + ok := detectMPPStore(ctx, t.tikvClient, t.address, detectTimeoutLimit) + if !ok { + metrics.TiFlashFailedMPPStoreState.WithLabelValues(t.address).Set(1) + t.lock.recoveryTime = time.Time{} // if detect failed,reset recovery time to zero. + return + } + + // record the time of the first recovery + if t.lock.recoveryTime.IsZero() { + t.lock.recoveryTime = time.Now() + } +} + +func (t *MPPStoreState) isRecovery(ctx context.Context, recoveryTTL time.Duration) bool { + if !t.lock.TryLock() { + return false + } + defer t.lock.Unlock() + + t.lock.lastLookupTime = time.Now() + if !t.lock.recoveryTime.IsZero() && time.Since(t.lock.recoveryTime) > recoveryTTL { + return true + } + logutil.Logger(ctx).Debug("Cannot detect store's availability "+ + "because the current time has not recovery or wait mppStoreFailTTL", + zap.String("store address", t.address), + zap.Time("recovery time", t.lock.recoveryTime), + zap.Duration("MPPStoreFailTTL", recoveryTTL)) + return false +} + +func (t MPPFailedStoreProber) scan(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + logutil.Logger(ctx).Warn("mpp failed store probe scan error,will restart", zap.Any("recover", r), zap.Stack("stack")) + } + }() + + do := func(k, v any) { + address := fmt.Sprint(k) + state, ok := v.(*MPPStoreState) + if !ok { + logutil.BgLogger().Warn("MPPStoreState struct assert failed,will be clean", + zap.String("address", address)) + t.Delete(address) + return + } + + if !state.lock.TryLock() { + return + } + defer state.lock.Unlock() + + state.detect(ctx, t.detectPeriod, t.detectTimeoutLimit) + + // clean restored store + if !state.lock.recoveryTime.IsZero() && time.Since(state.lock.recoveryTime) > t.maxRecoveryTimeLimit { + t.Delete(address) + // clean store that may be obsolete + } else if state.lock.recoveryTime.IsZero() && time.Since(state.lock.lastLookupTime) > t.maxObsoletTimeLimit { + t.Delete(address) + } + } + + f := func(k, v any) bool { + go do(k, v) + return true + } + + metrics.TiFlashFailedMPPStoreState.WithLabelValues("probe").Set(-1) //probe heartbeat + t.failedMPPStores.Range(f) +} + +// Add add a store when sync probe failed +func (t *MPPFailedStoreProber) Add(ctx context.Context, address string, tikvClient tikv.Client) { + state := MPPStoreState{ + address: address, + tikvClient: tikvClient, + } + state.lock.lastLookupTime = time.Now() + logutil.Logger(ctx).Debug("add mpp store to failed list", zap.String("address", address)) + t.failedMPPStores.Store(address, &state) +} + +// IsRecovery check whether the store is recovery +func (t *MPPFailedStoreProber) IsRecovery(ctx context.Context, address string, recoveryTTL time.Duration) bool { + logutil.Logger(ctx).Debug("check failed store recovery", + zap.String("address", address), zap.Duration("ttl", recoveryTTL)) + v, ok := t.failedMPPStores.Load(address) + if !ok { + // store not in failed map + return true + } + + state, ok := v.(*MPPStoreState) + if !ok { + logutil.BgLogger().Warn("MPPStoreState struct assert failed,will be clean", + zap.String("address", address)) + t.Delete(address) + return false + } + + return state.isRecovery(ctx, recoveryTTL) +} + +// Run a loop of scan +// there can be only one background task +func (t *MPPFailedStoreProber) Run() { + if !t.lock.TryLock() { + return + } + t.wg.Add(1) + t.isStop.Swap(false) + go func() { + defer t.wg.Done() + defer t.lock.Unlock() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for { + select { + case <-t.ctx.Done(): + logutil.BgLogger().Debug("ctx.done") + return + case <-ticker.C: + t.scan(t.ctx) + } + } + }() + logutil.BgLogger().Debug("run a background probe process for mpp") +} + +// Stop stop background goroutine +func (t *MPPFailedStoreProber) Stop() { + if !t.isStop.CompareAndSwap(false, true) { + return + } + t.cancel() + t.wg.Wait() + logutil.BgLogger().Debug("stop background task") +} + +// Delete clean store from failed map +func (t *MPPFailedStoreProber) Delete(address string) { + metrics.TiFlashFailedMPPStoreState.DeleteLabelValues(address) + _, ok := t.failedMPPStores.LoadAndDelete(address) + if !ok { + logutil.BgLogger().Warn("Store is deleted", zap.String("address", address)) + } +} + +// MPPStore detect function +func detectMPPStore(ctx context.Context, client tikv.Client, address string, detectTimeoutLimit time.Duration) bool { + resp, err := client.SendRequest(ctx, address, &tikvrpc.Request{ + Type: tikvrpc.CmdMPPAlive, + StoreTp: tikvrpc.TiFlash, + Req: &mpp.IsAliveRequest{}, + Context: kvrpcpb.Context{}, + }, detectTimeoutLimit) + if err != nil || !resp.Resp.(*mpp.IsAliveResponse).Available { + if err == nil { + err = fmt.Errorf("store not ready to serve") + } + logutil.BgLogger().Warn("Store is not ready", + zap.String("store address", address), + zap.String("err message", err.Error())) + return false + } + return true +} + +func init() { + ctx, cancel := context.WithCancel(context.Background()) + isStop := atomic.Bool{} + isStop.Swap(true) + GlobalMPPFailedStoreProber = &MPPFailedStoreProber{ + failedMPPStores: &sync.Map{}, + lock: &sync.Mutex{}, + isStop: &isStop, + ctx: ctx, + cancel: cancel, + wg: &sync.WaitGroup{}, + detectPeriod: DetectPeriod, + detectTimeoutLimit: DetectTimeoutLimit, + maxRecoveryTimeLimit: MaxRecoveryTimeLimit, + maxObsoletTimeLimit: MaxObsoletTimeLimit, + } +} diff --git a/store/copr/mpp_probe_test.go b/store/copr/mpp_probe_test.go new file mode 100644 index 0000000000000..7826c970d3e1e --- /dev/null +++ b/store/copr/mpp_probe_test.go @@ -0,0 +1,177 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package copr + +import ( + "context" + "testing" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/mpp" + "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/tikvrpc" +) + +const ( + testimeout = "timeout" + Error = "error" + Normal = "normal" +) + +type mockDetectClient struct { + errortestype string +} + +func (t *mockDetectClient) CloseAddr(string) error { + return nil +} + +func (t *mockDetectClient) Close() error { + return nil +} + +func (t *mockDetectClient) SendRequest( + ctx context.Context, + addr string, + req *tikvrpc.Request, + timeout time.Duration, +) (*tikvrpc.Response, error) { + if t.errortestype == Error { + return nil, errors.New("store error") + } else if t.errortestype == testimeout { + return &tikvrpc.Response{Resp: &mpp.IsAliveResponse{}}, nil + } + + return &tikvrpc.Response{Resp: &mpp.IsAliveResponse{Available: true}}, nil +} + +type ProbeTest map[string]*mockDetectClient + +func (t ProbeTest) add(ctx context.Context) { + for k, v := range t { + GlobalMPPFailedStoreProber.Add(ctx, k, v) + } +} + +func (t ProbeTest) reSetErrortestype(to string) { + for k, v := range t { + if to == Normal { + v.errortestype = Normal + } else { + v.errortestype = k + } + } +} + +func (t ProbeTest) judge(ctx context.Context, test *testing.T, recoveryTTL time.Duration, need bool) { + for k := range t { + ok := GlobalMPPFailedStoreProber.IsRecovery(ctx, k, recoveryTTL) + require.Equal(test, need, ok) + } +} + +func failedStoreSizeJudge(ctx context.Context, test *testing.T, need int) { + var l int + GlobalMPPFailedStoreProber.scan(ctx) + time.Sleep(time.Second / 10) + GlobalMPPFailedStoreProber.failedMPPStores.Range(func(k, v interface{}) bool { + l++ + return true + }) + require.Equal(test, need, l) +} + +func testFlow(ctx context.Context, probetestest ProbeTest, test *testing.T, flow []string) { + probetestest.add(ctx) + for _, to := range flow { + probetestest.reSetErrortestype(to) + + GlobalMPPFailedStoreProber.scan(ctx) + time.Sleep(time.Second / 10) //wait detect goroutine finish + + var need bool + if to == Normal { + need = true + } + probetestest.judge(ctx, test, 0, need) + probetestest.judge(ctx, test, time.Minute, false) + } + + lastTo := flow[len(flow)-1] + cleanRecover := func(need int) { + GlobalMPPFailedStoreProber.maxRecoveryTimeLimit = 0 - time.Second + failedStoreSizeJudge(ctx, test, need) + GlobalMPPFailedStoreProber.maxRecoveryTimeLimit = MaxRecoveryTimeLimit + } + + cleanObsolet := func(need int) { + GlobalMPPFailedStoreProber.maxObsoletTimeLimit = 0 - time.Second + failedStoreSizeJudge(ctx, test, need) + GlobalMPPFailedStoreProber.maxObsoletTimeLimit = MaxObsoletTimeLimit + } + + if lastTo == Error { + cleanRecover(2) + cleanObsolet(0) + } else if lastTo == Normal { + cleanObsolet(2) + cleanRecover(0) + } +} + +func TestMPPFailedStoreProbe(t *testing.T) { + ctx := context.Background() + + notExistAddress := "not exist address" + + GlobalMPPFailedStoreProber.detectPeriod = 0 - time.Second + + // check not exist address + ok := GlobalMPPFailedStoreProber.IsRecovery(ctx, notExistAddress, 0) + require.True(t, ok) + + GlobalMPPFailedStoreProber.scan(ctx) + + probetestest := map[string]*mockDetectClient{ + testimeout: {errortestype: testimeout}, + Error: {errortestype: Error}, + } + + testFlowFinallyRecover := []string{Error, Normal, Error, Error, Normal} + testFlow(ctx, probetestest, t, testFlowFinallyRecover) + testFlowFinallyDesert := []string{Error, Normal, Normal, Error, Error} + testFlow(ctx, probetestest, t, testFlowFinallyDesert) +} + +func TestMPPFailedStoreProbeGoroutineTask(t *testing.T) { + // Confirm that multiple tasks are not allowed + GlobalMPPFailedStoreProber.lock.Lock() + GlobalMPPFailedStoreProber.Run() + GlobalMPPFailedStoreProber.lock.Unlock() + + GlobalMPPFailedStoreProber.Run() + GlobalMPPFailedStoreProber.Stop() +} + +func TestMPPFailedStoreAssertFailed(t *testing.T) { + ctx := context.Background() + + GlobalMPPFailedStoreProber.failedMPPStores.Store("errorinfo", nil) + GlobalMPPFailedStoreProber.scan(ctx) + + GlobalMPPFailedStoreProber.failedMPPStores.Store("errorinfo", nil) + GlobalMPPFailedStoreProber.IsRecovery(ctx, "errorinfo", 0) +} diff --git a/store/copr/store.go b/store/copr/store.go index 758109b81d805..32553961acc67 100644 --- a/store/copr/store.go +++ b/store/copr/store.go @@ -84,6 +84,7 @@ func NewStore(s *tikv.KVStore, coprCacheConfig *config.CoprocessorCache) (*Store if err != nil { return nil, errors.Trace(err) } + /* #nosec G404 */ return &Store{ kvStore: &kvStore{store: s}, diff --git a/table/column.go b/table/column.go index 0a80ddae4e8d4..c63febe75b0ba 100644 --- a/table/column.go +++ b/table/column.go @@ -323,6 +323,7 @@ func CastValue(ctx sessionctx.Context, val types.Datum, col *model.ColumnInfo, r } err = sc.HandleTruncate(err) + err = sc.HandleOverflow(err, err) if forceIgnoreTruncate { err = nil diff --git a/testkit/asynctestkit.go b/testkit/asynctestkit.go index aa0f3fcadf8ef..a875088c82abf 100644 --- a/testkit/asynctestkit.go +++ b/testkit/asynctestkit.go @@ -183,6 +183,21 @@ func (tk *AsyncTestKit) MustExec(ctx context.Context, sql string, args ...interf } } +// MustGetErrMsg executes a sql statement and assert its error message. +func (tk *AsyncTestKit) MustGetErrMsg(ctx context.Context, sql string, errStr string) { + err := tk.ExecToErr(ctx, sql) + tk.require.EqualError(err, errStr) +} + +// ExecToErr executes a sql statement and discard results. +func (tk *AsyncTestKit) ExecToErr(ctx context.Context, sql string, args ...interface{}) error { + res, err := tk.Exec(ctx, sql, args...) + if res != nil { + tk.require.NoError(res.Close()) + } + return err +} + // MustQuery query the statements and returns result rows. // If expected result is set it asserts the query result equals expected result. func (tk *AsyncTestKit) MustQuery(ctx context.Context, sql string, args ...interface{}) *Result { diff --git a/testkit/mocksessionmanager.go b/testkit/mocksessionmanager.go index a9e4d085dc34d..550ff69132d91 100644 --- a/testkit/mocksessionmanager.go +++ b/testkit/mocksessionmanager.go @@ -18,6 +18,7 @@ import ( "crypto/tls" "sync" + "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" @@ -31,6 +32,7 @@ type MockSessionManager struct { PSMu sync.RWMutex SerID uint64 TxnInfo []*txninfo.TxnInfo + Dom *domain.Domain conn map[uint64]session.Session mu sync.Mutex } @@ -68,6 +70,11 @@ func (msm *MockSessionManager) ShowProcessList() map[uint64]*util.ProcessInfo { ret[connID] = pi.ShowProcess() } msm.mu.Unlock() + if msm.Dom != nil { + for connID, pi := range msm.Dom.SysProcTracker().GetSysProcessList() { + ret[connID] = pi + } + } return ret } @@ -85,6 +92,11 @@ func (msm *MockSessionManager) GetProcessInfo(id uint64) (*util.ProcessInfo, boo if sess := msm.conn[id]; sess != nil { return sess.ShowProcess(), true } + if msm.Dom != nil { + if pinfo, ok := msm.Dom.SysProcTracker().GetSysProcessList()[id]; ok { + return pinfo, true + } + } return &util.ProcessInfo{}, false } diff --git a/tests/realtikvtest/sessiontest/session_fail_test.go b/tests/realtikvtest/sessiontest/session_fail_test.go index 919932e6f9357..a3df51be821c0 100644 --- a/tests/realtikvtest/sessiontest/session_fail_test.go +++ b/tests/realtikvtest/sessiontest/session_fail_test.go @@ -200,7 +200,6 @@ func TestAutoCommitNeedNotLinearizability(t *testing.T) { func TestKill(t *testing.T) { store := realtikvtest.CreateMockStoreAndSetup(t) - tk := testkit.NewTestKit(t, store) tk.MustExec("kill connection_id();") } diff --git a/tidb-server/BUILD.bazel b/tidb-server/BUILD.bazel index 493eedfcccbfa..361a929351642 100644 --- a/tidb-server/BUILD.bazel +++ b/tidb-server/BUILD.bazel @@ -29,6 +29,7 @@ go_library( "//sessionctx/variable", "//statistics", "//store", + "//store/copr", "//store/driver", "//store/mockstore", "//store/mockstore/unistore/metrics", diff --git a/tidb-server/main.go b/tidb-server/main.go index d58e24a531f4c..8605c7d0c3aaa 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -54,6 +54,7 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics" kvstore "github.com/pingcap/tidb/store" + "github.com/pingcap/tidb/store/copr" "github.com/pingcap/tidb/store/driver" "github.com/pingcap/tidb/store/mockstore" uni_metrics "github.com/pingcap/tidb/store/mockstore/unistore/metrics" @@ -309,6 +310,7 @@ func createStoreAndDomain() (kv.Storage, *domain.Domain) { var err error storage, err := kvstore.New(fullPath) terror.MustNil(err) + copr.GlobalMPPFailedStoreProber.Run() err = infosync.CheckTiKVVersion(storage, *semver.New(versioninfo.TiKVMinVersion)) terror.MustNil(err) // Bootstrap a session to load information schema. @@ -814,6 +816,7 @@ func setupTracing() { func closeDomainAndStorage(storage kv.Storage, dom *domain.Domain) { tikv.StoreShuttingDown(1) dom.Close() + copr.GlobalMPPFailedStoreProber.Stop() err := storage.Close() terror.Log(errors.Trace(err)) } diff --git a/ttl/cache/ttlstatus.go b/ttl/cache/ttlstatus.go index 1657105e6c3e7..f14c1a559ae95 100644 --- a/ttl/cache/ttlstatus.go +++ b/ttl/cache/ttlstatus.go @@ -16,7 +16,6 @@ package cache import ( "context" - "fmt" "time" "github.com/pingcap/tidb/sessionctx" @@ -43,8 +42,8 @@ const ( const selectFromTTLTableStatus = "SELECT LOW_PRIORITY table_id,parent_table_id,table_statistics,last_job_id,last_job_start_time,last_job_finish_time,last_job_ttl_expire,last_job_summary,current_job_id,current_job_owner_id,current_job_owner_addr,current_job_owner_hb_time,current_job_start_time,current_job_ttl_expire,current_job_state,current_job_status,current_job_status_update_time FROM mysql.tidb_ttl_table_status" // SelectFromTTLTableStatusWithID returns an SQL statement to get the table status from table id -func SelectFromTTLTableStatusWithID(tableID int64) string { - return selectFromTTLTableStatus + fmt.Sprintf(" WHERE table_id = %d", tableID) +func SelectFromTTLTableStatusWithID(tableID int64) (string, []interface{}) { + return selectFromTTLTableStatus + " WHERE table_id = %?", []interface{}{tableID} } // TableStatus contains the corresponding information in the system table `mysql.tidb_ttl_table_status` diff --git a/ttl/ttlworker/BUILD.bazel b/ttl/ttlworker/BUILD.bazel index 8d031e22e6e4a..527c782ed691e 100644 --- a/ttl/ttlworker/BUILD.bazel +++ b/ttl/ttlworker/BUILD.bazel @@ -51,6 +51,7 @@ go_test( flaky = True, deps = [ "//infoschema", + "//kv", "//parser/ast", "//parser/model", "//parser/mysql", diff --git a/ttl/ttlworker/job.go b/ttl/ttlworker/job.go index 1a2f351465027..8628e9a6c9746 100644 --- a/ttl/ttlworker/job.go +++ b/ttl/ttlworker/job.go @@ -17,7 +17,6 @@ package ttlworker import ( "context" "encoding/json" - "fmt" "sync" "time" @@ -29,20 +28,34 @@ import ( "go.uber.org/zap" ) -const updateJobCurrentStatusTemplate = "UPDATE mysql.tidb_ttl_table_status SET current_job_status = '%s' WHERE table_id = %d AND current_job_status = '%s' AND current_job_id = '%s'" -const finishJobTemplate = "UPDATE mysql.tidb_ttl_table_status SET last_job_id = current_job_id, last_job_start_time = current_job_start_time, last_job_finish_time = '%s', last_job_ttl_expire = current_job_ttl_expire, last_job_summary = '%s', current_job_id = NULL, current_job_owner_id = NULL, current_job_owner_hb_time = NULL, current_job_start_time = NULL, current_job_ttl_expire = NULL, current_job_state = NULL, current_job_status = NULL, current_job_status_update_time = NULL WHERE table_id = %d AND current_job_id = '%s'" -const updateJobStateTemplate = "UPDATE mysql.tidb_ttl_table_status SET current_job_state = '%s' WHERE table_id = %d AND current_job_id = '%s' AND current_job_owner_id = '%s'" +const updateJobCurrentStatusTemplate = "UPDATE mysql.tidb_ttl_table_status SET current_job_status = %? WHERE table_id = %? AND current_job_status = %? AND current_job_id = %?" +const finishJobTemplate = `UPDATE mysql.tidb_ttl_table_status + SET last_job_id = current_job_id, + last_job_start_time = current_job_start_time, + last_job_finish_time = %?, + last_job_ttl_expire = current_job_ttl_expire, + last_job_summary = %?, + current_job_id = NULL, + current_job_owner_id = NULL, + current_job_owner_hb_time = NULL, + current_job_start_time = NULL, + current_job_ttl_expire = NULL, + current_job_state = NULL, + current_job_status = NULL, + current_job_status_update_time = NULL + WHERE table_id = %? AND current_job_id = %?` +const updateJobStateTemplate = "UPDATE mysql.tidb_ttl_table_status SET current_job_state = %? WHERE table_id = %? AND current_job_id = %? AND current_job_owner_id = %?" -func updateJobCurrentStatusSQL(tableID int64, oldStatus cache.JobStatus, newStatus cache.JobStatus, jobID string) string { - return fmt.Sprintf(updateJobCurrentStatusTemplate, newStatus, tableID, oldStatus, jobID) +func updateJobCurrentStatusSQL(tableID int64, oldStatus cache.JobStatus, newStatus cache.JobStatus, jobID string) (string, []interface{}) { + return updateJobCurrentStatusTemplate, []interface{}{newStatus, tableID, oldStatus, jobID} } -func finishJobSQL(tableID int64, finishTime time.Time, summary string, jobID string) string { - return fmt.Sprintf(finishJobTemplate, finishTime.Format(timeFormat), summary, tableID, jobID) +func finishJobSQL(tableID int64, finishTime time.Time, summary string, jobID string) (string, []interface{}) { + return finishJobTemplate, []interface{}{finishTime.Format(timeFormat), summary, tableID, jobID} } -func updateJobState(tableID int64, currentJobID string, currentJobState string, currentJobOwnerID string) string { - return fmt.Sprintf(updateJobStateTemplate, currentJobState, tableID, currentJobID, currentJobOwnerID) +func updateJobState(tableID int64, currentJobID string, currentJobState string, currentJobOwnerID string) (string, []interface{}) { + return updateJobStateTemplate, []interface{}{currentJobState, tableID, currentJobID, currentJobOwnerID} } type ttlJob struct { @@ -76,9 +89,10 @@ func (job *ttlJob) changeStatus(ctx context.Context, se session.Session, status job.status = status job.statusMutex.Unlock() - _, err := se.ExecuteSQL(ctx, updateJobCurrentStatusSQL(job.tbl.ID, oldStatus, status, job.id)) + sql, args := updateJobCurrentStatusSQL(job.tbl.ID, oldStatus, status, job.id) + _, err := se.ExecuteSQL(ctx, sql, args...) if err != nil { - return errors.Trace(err) + return errors.Wrapf(err, "execute sql: %s", sql) } return nil @@ -89,9 +103,10 @@ func (job *ttlJob) updateState(ctx context.Context, se session.Session) error { if err != nil { logutil.Logger(job.ctx).Warn("fail to generate summary for ttl job", zap.Error(err)) } - _, err = se.ExecuteSQL(ctx, updateJobState(job.tbl.ID, job.id, summary, job.ownerID)) + sql, args := updateJobState(job.tbl.ID, job.id, summary, job.ownerID) + _, err = se.ExecuteSQL(ctx, sql, args...) if err != nil { - return errors.Trace(err) + return errors.Wrapf(err, "execute sql: %s", sql) } return nil @@ -115,9 +130,10 @@ func (job *ttlJob) finish(se session.Session, now time.Time) { } // at this time, the job.ctx may have been canceled (to cancel this job) // even when it's canceled, we'll need to update the states, so use another context - _, err = se.ExecuteSQL(context.TODO(), finishJobSQL(job.tbl.ID, now, summary, job.id)) + sql, args := finishJobSQL(job.tbl.ID, now, summary, job.id) + _, err = se.ExecuteSQL(context.TODO(), sql, args...) if err != nil { - logutil.Logger(job.ctx).Error("fail to finish a ttl job", zap.Error(err), zap.Int64("tableID", job.tbl.ID), zap.String("jobID", job.id)) + logutil.Logger(job.ctx).Error("fail to finish a ttl job", zap.Error(err), zap.Int64("tableID", job.tbl.ID), zap.String("jobID", job.id), zap.String("sql", sql), zap.Any("arguments", args)) } } diff --git a/ttl/ttlworker/job_manager.go b/ttl/ttlworker/job_manager.go index 58c47d05efe4f..25af41e46ca58 100644 --- a/ttl/ttlworker/job_manager.go +++ b/ttl/ttlworker/job_manager.go @@ -16,7 +16,6 @@ package ttlworker import ( "context" - "fmt" "time" "github.com/pingcap/errors" @@ -31,22 +30,30 @@ import ( "go.uber.org/zap" ) -const insertNewTableIntoStatusTemplate = "INSERT INTO mysql.tidb_ttl_table_status (table_id,parent_table_id) VALUES (%d, %d)" -const setTableStatusOwnerTemplate = "UPDATE mysql.tidb_ttl_table_status SET current_job_id = UUID(), current_job_owner_id = '%s',current_job_start_time = '%s',current_job_status = 'waiting',current_job_status_update_time = '%s',current_job_ttl_expire = '%s',current_job_owner_hb_time = '%s' WHERE table_id = %d" -const updateHeartBeatTemplate = "UPDATE mysql.tidb_ttl_table_status SET current_job_owner_hb_time = '%s' WHERE table_id = %d AND current_job_owner_id = '%s'" +const insertNewTableIntoStatusTemplate = "INSERT INTO mysql.tidb_ttl_table_status (table_id,parent_table_id) VALUES (%?, %?)" +const setTableStatusOwnerTemplate = `UPDATE mysql.tidb_ttl_table_status + SET current_job_id = UUID(), + current_job_owner_id = %?, + current_job_start_time = %?, + current_job_status = 'waiting', + current_job_status_update_time = %?, + current_job_ttl_expire = %?, + current_job_owner_hb_time = %? + WHERE table_id = %?` +const updateHeartBeatTemplate = "UPDATE mysql.tidb_ttl_table_status SET current_job_owner_hb_time = %? WHERE table_id = %? AND current_job_owner_id = %?" const timeFormat = "2006-01-02 15:04:05" -func insertNewTableIntoStatusSQL(tableID int64, parentTableID int64) string { - return fmt.Sprintf(insertNewTableIntoStatusTemplate, tableID, parentTableID) +func insertNewTableIntoStatusSQL(tableID int64, parentTableID int64) (string, []interface{}) { + return insertNewTableIntoStatusTemplate, []interface{}{tableID, parentTableID} } -func setTableStatusOwnerSQL(tableID int64, now time.Time, currentJobTTLExpire time.Time, id string) string { - return fmt.Sprintf(setTableStatusOwnerTemplate, id, now.Format(timeFormat), now.Format(timeFormat), currentJobTTLExpire.Format(timeFormat), now.Format(timeFormat), tableID) +func setTableStatusOwnerSQL(tableID int64, now time.Time, currentJobTTLExpire time.Time, id string) (string, []interface{}) { + return setTableStatusOwnerTemplate, []interface{}{id, now.Format(timeFormat), now.Format(timeFormat), currentJobTTLExpire.Format(timeFormat), now.Format(timeFormat), tableID} } -func updateHeartBeatSQL(tableID int64, now time.Time, id string) string { - return fmt.Sprintf(updateHeartBeatTemplate, now.Format(timeFormat), tableID, id) +func updateHeartBeatSQL(tableID int64, now time.Time, id string) (string, []interface{}) { + return updateHeartBeatTemplate, []interface{}{now.Format(timeFormat), tableID, id} } // JobManager schedules and manages the ttl jobs on this instance @@ -503,19 +510,22 @@ func (m *JobManager) lockNewJob(ctx context.Context, se session.Session, table * var expireTime time.Time err := se.RunInTxn(ctx, func() error { - rows, err := se.ExecuteSQL(ctx, cache.SelectFromTTLTableStatusWithID(table.ID)) + sql, args := cache.SelectFromTTLTableStatusWithID(table.ID) + rows, err := se.ExecuteSQL(ctx, sql, args...) if err != nil { - return err + return errors.Wrapf(err, "execute sql: %s", sql) } if len(rows) == 0 { // cannot find the row, insert the status row - _, err = se.ExecuteSQL(ctx, insertNewTableIntoStatusSQL(table.ID, table.TableInfo.ID)) + sql, args := insertNewTableIntoStatusSQL(table.ID, table.TableInfo.ID) + _, err = se.ExecuteSQL(ctx, sql, args...) if err != nil { - return err + return errors.Wrapf(err, "execute sql: %s", sql) } - rows, err = se.ExecuteSQL(ctx, cache.SelectFromTTLTableStatusWithID(table.ID)) + sql, args = cache.SelectFromTTLTableStatusWithID(table.ID) + rows, err = se.ExecuteSQL(ctx, sql, args...) if err != nil { - return err + return errors.Wrapf(err, "execute sql: %s", sql) } if len(rows) == 0 { return errors.New("table status row still doesn't exist after insertion") @@ -534,9 +544,9 @@ func (m *JobManager) lockNewJob(ctx context.Context, se session.Session, table * return err } - _, err = se.ExecuteSQL(ctx, setTableStatusOwnerSQL(table.ID, now, expireTime, m.id)) - - return err + sql, args = setTableStatusOwnerSQL(table.ID, now, expireTime, m.id) + _, err = se.ExecuteSQL(ctx, sql, args...) + return errors.Wrapf(err, "execute sql: %s", sql) }) if err != nil { return nil, err @@ -599,9 +609,10 @@ func (m *JobManager) createNewJob(expireTime time.Time, now time.Time, table *ca func (m *JobManager) updateHeartBeat(ctx context.Context, se session.Session) error { now := se.Now() for _, job := range m.localJobs() { - _, err := se.ExecuteSQL(ctx, updateHeartBeatSQL(job.tbl.ID, now, m.id)) + sql, args := updateHeartBeatSQL(job.tbl.ID, now, m.id) + _, err := se.ExecuteSQL(ctx, sql, args...) if err != nil { - return errors.Trace(err) + return errors.Wrapf(err, "execute sql: %s", sql) } // also updates some internal state for this job err = job.updateState(ctx, se) diff --git a/ttl/ttlworker/job_manager_integration_test.go b/ttl/ttlworker/job_manager_integration_test.go index 8c299afcd48de..8d33a736d2b9d 100644 --- a/ttl/ttlworker/job_manager_integration_test.go +++ b/ttl/ttlworker/job_manager_integration_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "github.com/pingcap/errors" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/model" dbsession "github.com/pingcap/tidb/session" @@ -35,10 +37,8 @@ import ( "go.uber.org/zap" ) -func TestParallelLockNewJob(t *testing.T) { - store := testkit.CreateMockStore(t) - - sessionFactory := func() session.Session { +func sessionFactory(t *testing.T, store kv.Storage) func() session.Session { + return func() session.Session { dbSession, err := dbsession.CreateSession4Test(store) require.NoError(t, err) se := session.NewSession(dbSession, dbSession, nil) @@ -50,6 +50,12 @@ func TestParallelLockNewJob(t *testing.T) { return se } +} + +func TestParallelLockNewJob(t *testing.T) { + store := testkit.CreateMockStore(t) + + sessionFactory := sessionFactory(t, store) storedTTLJobRunInterval := variable.TTLJobRunInterval.Load() variable.TTLJobRunInterval.Store(0) @@ -96,3 +102,24 @@ func TestParallelLockNewJob(t *testing.T) { successJob.Finish(se, time.Now()) } } + +func TestFinishJob(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + + sessionFactory := sessionFactory(t, store) + + testTable := &cache.PhysicalTable{ID: 2, TableInfo: &model.TableInfo{ID: 1, TTLInfo: &model.TTLInfo{IntervalExprStr: "1", IntervalTimeUnit: int(ast.TimeUnitDay)}}} + + tk.MustExec("insert into mysql.tidb_ttl_table_status(table_id) values (2)") + + // finish with error + m := ttlworker.NewJobManager("test-id", nil, store) + se := sessionFactory() + job, err := m.LockNewJob(context.Background(), se, testTable, time.Now()) + require.NoError(t, err) + job.SetScanErr(errors.New(`"'an error message contains both single and double quote'"`)) + job.Finish(se, time.Now()) + + tk.MustQuery("select table_id, last_job_summary from mysql.tidb_ttl_table_status").Check(testkit.Rows("2 {\"total_rows\":0,\"success_rows\":0,\"error_rows\":0,\"total_scan_task\":1,\"scheduled_scan_task\":0,\"finished_scan_task\":0,\"scan_task_err\":\"\\\"'an error message contains both single and double quote'\\\"\"}")) +} diff --git a/ttl/ttlworker/job_manager_test.go b/ttl/ttlworker/job_manager_test.go index 6718c384543fe..1f649abb065af 100644 --- a/ttl/ttlworker/job_manager_test.go +++ b/ttl/ttlworker/job_manager_test.go @@ -156,6 +156,10 @@ func (j *ttlJob) ID() string { return j.id } +func (j *ttlJob) SetScanErr(err error) { + j.scanTaskErr = err +} + func newMockTTLJob(tbl *cache.PhysicalTable, status cache.JobStatus) *ttlJob { statistics := &ttlStatistics{} return &ttlJob{tbl: tbl, ctx: context.Background(), statistics: statistics, status: status, tasks: []*ttlScanTask{{ctx: context.Background(), tbl: tbl, statistics: statistics}}} @@ -216,8 +220,18 @@ func TestLockNewTable(t *testing.T) { testPhysicalTable := &cache.PhysicalTable{ID: 1, TableInfo: &model.TableInfo{ID: 1, TTLInfo: &model.TTLInfo{ColumnName: model.NewCIStr("test"), IntervalExprStr: "5 Year"}}} + type executeInfo struct { + sql string + args []interface{} + } + getExecuteInfo := func(sql string, args []interface{}) executeInfo { + return executeInfo{ + sql, + args, + } + } type sqlExecute struct { - sql string + executeInfo rows []chunk.Row err error @@ -231,47 +245,47 @@ func TestLockNewTable(t *testing.T) { }{ {"normal lock table", testPhysicalTable, []sqlExecute{ { - cache.SelectFromTTLTableStatusWithID(1), + getExecuteInfo(cache.SelectFromTTLTableStatusWithID(1)), newTTLTableStatusRows(&cache.TableStatus{TableID: 1}), nil, }, { - setTableStatusOwnerSQL(1, now, expireTime, "test-id"), + getExecuteInfo(setTableStatusOwnerSQL(1, now, expireTime, "test-id")), nil, nil, }, { - updateStatusSQL, + getExecuteInfo(updateStatusSQL, nil), newTTLTableStatusRows(&cache.TableStatus{TableID: 1}), nil, }, }, true, false}, {"select nothing", testPhysicalTable, []sqlExecute{ { - cache.SelectFromTTLTableStatusWithID(1), + getExecuteInfo(cache.SelectFromTTLTableStatusWithID(1)), nil, nil, }, { - insertNewTableIntoStatusSQL(1, 1), + getExecuteInfo(insertNewTableIntoStatusSQL(1, 1)), nil, nil, }, { - cache.SelectFromTTLTableStatusWithID(1), + getExecuteInfo(cache.SelectFromTTLTableStatusWithID(1)), newTTLTableStatusRows(&cache.TableStatus{TableID: 1}), nil, }, { - setTableStatusOwnerSQL(1, now, expireTime, "test-id"), + getExecuteInfo(setTableStatusOwnerSQL(1, now, expireTime, "test-id")), nil, nil, }, { - updateStatusSQL, + getExecuteInfo(updateStatusSQL, nil), newTTLTableStatusRows(&cache.TableStatus{TableID: 1}), nil, }, }, true, false}, {"return error", testPhysicalTable, []sqlExecute{ { - cache.SelectFromTTLTableStatusWithID(1), + getExecuteInfo(cache.SelectFromTTLTableStatusWithID(1)), newTTLTableStatusRows(&cache.TableStatus{TableID: 1}), nil, }, { - setTableStatusOwnerSQL(1, now, expireTime, "test-id"), + getExecuteInfo(setTableStatusOwnerSQL(1, now, expireTime, "test-id")), nil, errors.New("test error message"), }, }, false, true}, @@ -287,6 +301,7 @@ func TestLockNewTable(t *testing.T) { se.executeSQL = func(ctx context.Context, sql string, args ...interface{}) (rows []chunk.Row, err error) { assert.Less(t, sqlCounter, len(c.sqls)) assert.Equal(t, sql, c.sqls[sqlCounter].sql) + assert.Equal(t, args, c.sqls[sqlCounter].args) rows = c.sqls[sqlCounter].rows err = c.sqls[sqlCounter].err @@ -509,11 +524,26 @@ func TestCheckFinishedJob(t *testing.T) { m.runningJobs[0].taskIter = 1 m.runningJobs[0].finishedScanTaskCounter = 1 - m.checkFinishedJob(se, se.Now()) + // meetArg records whether the sql statement uses the arg + meetArg := false + now := se.Now() + jobID := m.runningJobs[0].id + se.executeSQL = func(ctx context.Context, sql string, args ...interface{}) ([]chunk.Row, error) { + if len(args) > 0 { + meetArg = true + expectedSQL, expectedArgs := finishJobSQL(tbl.ID, now, "{\"total_rows\":1,\"success_rows\":1,\"error_rows\":0,\"total_scan_task\":1,\"scheduled_scan_task\":1,\"finished_scan_task\":1}", jobID) + assert.Equal(t, expectedSQL, sql) + assert.Equal(t, expectedArgs, args) + } + return nil, nil + } + m.checkFinishedJob(se, now) assert.Len(t, m.runningJobs, 0) + assert.Equal(t, true, meetArg) + se.executeSQL = nil // check timeout job - now := se.Now() + now = se.Now() createTime := now.Add(-20 * time.Hour) m = NewJobManager("test-id", newMockSessionPool(t, tbl), nil) m.runningJobs = []*ttlJob{ diff --git a/ttl/ttlworker/session_test.go b/ttl/ttlworker/session_test.go index 877fd7996eaa7..712c696fa5f35 100644 --- a/ttl/ttlworker/session_test.go +++ b/ttl/ttlworker/session_test.go @@ -179,7 +179,7 @@ func (s *mockSession) ExecuteSQL(ctx context.Context, sql string, args ...interf } if s.executeSQL != nil { - return s.executeSQL(ctx, sql, args) + return s.executeSQL(ctx, sql, args...) } return s.rows, s.execErr } diff --git a/types/field_type_builder.go b/types/field_type_builder.go index 7c9f3bdc3177d..81554c4585442 100644 --- a/types/field_type_builder.go +++ b/types/field_type_builder.go @@ -114,6 +114,12 @@ func (b *FieldTypeBuilder) SetElems(elems []string) *FieldTypeBuilder { return b } +// SetArray sets array of the ft +func (b *FieldTypeBuilder) SetArray(x bool) *FieldTypeBuilder { + b.ft.SetArray(x) + return b +} + // Build returns the ft func (b *FieldTypeBuilder) Build() FieldType { return b.ft diff --git a/util/cpu/BUILD.bazel b/util/cpu/BUILD.bazel index 58bc047a332c4..08893520caaa0 100644 --- a/util/cpu/BUILD.bazel +++ b/util/cpu/BUILD.bazel @@ -20,5 +20,6 @@ go_test( name = "cpu_test", srcs = ["cpu_test.go"], embed = [":cpu"], + flaky = True, deps = ["@com_github_stretchr_testify//require"], ) diff --git a/util/cpu/cpu.go b/util/cpu/cpu.go index 416b3c3eaeb99..2803b4e106c49 100644 --- a/util/cpu/cpu.go +++ b/util/cpu/cpu.go @@ -56,11 +56,13 @@ func NewCPUObserver() *Observer { // Start starts the cpu observer. func (c *Observer) Start() { - ticker := time.NewTicker(100 * time.Millisecond) - defer ticker.Stop() c.wg.Add(1) go func() { - defer c.wg.Done() + ticker := time.NewTicker(100 * time.Millisecond) + defer func() { + ticker.Stop() + c.wg.Done() + }() for { select { case <-ticker.C: diff --git a/util/cpu/cpu_test.go b/util/cpu/cpu_test.go index 6c7e863f9060a..cd330a11e5196 100644 --- a/util/cpu/cpu_test.go +++ b/util/cpu/cpu_test.go @@ -42,9 +42,10 @@ func TestCPUValue(t *testing.T) { } }() } - time.Sleep(30 * time.Second) - require.Greater(t, Observer.observe(), 0.0) - require.Less(t, Observer.observe(), 1.0) + Observer.Start() + time.Sleep(5 * time.Second) + require.GreaterOrEqual(t, GetCPUUsage(), 0.0) + require.Less(t, GetCPUUsage(), 1.0) Observer.Stop() close(exit) wg.Wait() diff --git a/util/dbterror/ddl_terror.go b/util/dbterror/ddl_terror.go index 6c80d60ad6ec2..5db0d90c848e1 100644 --- a/util/dbterror/ddl_terror.go +++ b/util/dbterror/ddl_terror.go @@ -429,4 +429,7 @@ var ( ErrTempTableNotAllowedWithTTL = ClassDDL.NewStd(mysql.ErrTempTableNotAllowedWithTTL) // ErrUnsupportedTTLReferencedByFK returns when the TTL config is set for a table referenced by foreign key ErrUnsupportedTTLReferencedByFK = ClassDDL.NewStd(mysql.ErrUnsupportedTTLReferencedByFK) + + // ErrNotSupportedYet returns when tidb does not support this feature. + ErrNotSupportedYet = ClassDDL.NewStd(mysql.ErrNotSupportedYet) ) diff --git a/util/memory/tracker.go b/util/memory/tracker.go index 9c2adf31ace14..39261a45355a1 100644 --- a/util/memory/tracker.go +++ b/util/memory/tracker.go @@ -762,6 +762,17 @@ func (t *Tracker) CountAllChildrenMemUse() map[string]int64 { return trackerMemUseMap } +// GetChildrenForTest returns children trackers +func (t *Tracker) GetChildrenForTest() []*Tracker { + t.mu.Lock() + defer t.mu.Unlock() + trackers := make([]*Tracker, 0) + for _, list := range t.mu.children { + trackers = append(trackers, list...) + } + return trackers +} + func countChildMem(t *Tracker, familyTreeName string, trackerMemUseMap map[string]int64) { if len(familyTreeName) > 0 { familyTreeName += " <- " diff --git a/util/replayer/replayer.go b/util/replayer/replayer.go index 9fb0bc4f629fa..f89d26ec97717 100644 --- a/util/replayer/replayer.go +++ b/util/replayer/replayer.go @@ -33,13 +33,13 @@ type PlanReplayerTaskKey struct { } // GeneratePlanReplayerFile generates plan replayer file -func GeneratePlanReplayerFile() (*os.File, string, error) { +func GeneratePlanReplayerFile(isContinues bool) (*os.File, string, error) { path := GetPlanReplayerDirName() err := os.MkdirAll(path, os.ModePerm) if err != nil { return nil, "", errors.AddStack(err) } - fileName, err := generatePlanReplayerFileName() + fileName, err := generatePlanReplayerFileName(isContinues) if err != nil { return nil, "", errors.AddStack(err) } @@ -50,7 +50,7 @@ func GeneratePlanReplayerFile() (*os.File, string, error) { return zf, fileName, err } -func generatePlanReplayerFileName() (string, error) { +func generatePlanReplayerFileName(isContinues bool) (string, error) { // Generate key and create zip file time := time.Now().UnixNano() b := make([]byte, 16) @@ -60,6 +60,9 @@ func generatePlanReplayerFileName() (string, error) { return "", err } key := base64.URLEncoding.EncodeToString(b) + if isContinues { + return fmt.Sprintf("continues_replayer_%v_%v.zip", key, time), nil + } return fmt.Sprintf("replayer_%v_%v.zip", key, time), nil } diff --git a/util/sem/sem.go b/util/sem/sem.go index 161334c880a4a..e2364893b05b2 100644 --- a/util/sem/sem.go +++ b/util/sem/sem.go @@ -136,7 +136,6 @@ func IsInvisibleStatusVar(varName string) bool { func IsInvisibleSysVar(varNameInLower string) bool { switch varNameInLower { case variable.TiDBDDLSlowOprThreshold, // ddl_slow_threshold - variable.TiDBAllowRemoveAutoInc, variable.TiDBCheckMb4ValueInUTF8, variable.TiDBConfig, variable.TiDBEnableSlowLog, diff --git a/util/sem/sem_test.go b/util/sem/sem_test.go index cb0c47ad9225d..d19c2af27adcb 100644 --- a/util/sem/sem_test.go +++ b/util/sem/sem_test.go @@ -81,8 +81,8 @@ func TestIsInvisibleSysVar(t *testing.T) { assert.False(IsInvisibleSysVar(variable.Hostname)) // changes the value to default, but is not invisible assert.False(IsInvisibleSysVar(variable.TiDBEnableEnhancedSecurity)) // should be able to see the mode is on. + assert.False(IsInvisibleSysVar(variable.TiDBAllowRemoveAutoInc)) - assert.True(IsInvisibleSysVar(variable.TiDBAllowRemoveAutoInc)) assert.True(IsInvisibleSysVar(variable.TiDBCheckMb4ValueInUTF8)) assert.True(IsInvisibleSysVar(variable.TiDBConfig)) assert.True(IsInvisibleSysVar(variable.TiDBEnableSlowLog))