Merge branch 'master' into return-values

ddl/db_integration_test.go

@@ -1233,7 +1233,7 @@ func (s *testIntegrationSuite5) TestBackwardCompatibility(c *C) {
 
 	// Split the table.
 	tableStart := tablecodec.GenTableRecordPrefix(tbl.Meta().ID)
-	s.cluster.SplitKeys(tableStart, tableStart.PrefixNext(), 100)
+	s.cluster.SplitKeys(tableStart, tableStart.PrefixNext(), 10)
 
 	unique := false
 	indexName := model.NewCIStr("idx_b")
@@ -1275,7 +1275,6 @@ func (s *testIntegrationSuite5) TestBackwardCompatibility(c *C) {
 		historyJob, err := getHistoryDDLJob(s.store, job.ID)
 		c.Assert(err, IsNil)
 		if historyJob == nil {
-
 			continue
 		}
 		c.Assert(historyJob.Error, IsNil)

ddl/index.go

@@ -1217,10 +1217,9 @@ func (w *worker) updateReorgInfo(t table.PartitionedTable, reorg *reorgInfo) (bo
 
 	failpoint.Inject("mockUpdateCachedSafePoint", func(val failpoint.Value) {
 		if val.(bool) {
-			// 18 is for the logical time.
-			ts := oracle.GetPhysical(time.Now()) << 18
+			ts := oracle.GoTimeToTS(time.Now())
 			s := reorg.d.store.(tikv.Storage)
-			s.UpdateSPCache(uint64(ts), time.Now())
+			s.UpdateSPCache(ts, time.Now())
 			time.Sleep(time.Millisecond * 3)
 		}
 	})

docs/design/2021-04-26-lock-view.md

@@ -1,7 +1,7 @@
 # TiDB Design Documents
 
 - Author(s): [longfangsong](https://github.com/longfangsong), [MyonKeminta](http://github.com/MyonKeminta)
-- Last updated: May 6, 2021
+- Last updated: May 18, 2021
 - Discussion PR: N/A
 - Tracking Issue: https://github.com/pingcap/tidb/issues/24199
 
@@ -35,14 +35,14 @@ Several tables will be provided in `information_schema`. Some tables has both lo
 
 | Field      | Type       | Comment |
 |------------|------------|---------|
-|`TRX_ID`    | `unsigned bigint`   | The transaction ID (aka. start ts) |
-|`TRX_STARTED`|`time`| Human readable start time of the transaction |
-|`DIGEST`|`text`| The digest of the current executing SQL statement |
-|`SQLS` | `text` | A list of all executed SQL statements' digests |
-|`STATE`| `enum('Running', 'Lock waiting', 'Committing', 'RollingBack')`| The state of the transaction |
+| `TRX_ID`    | `unsigned bigint`   | The transaction ID (aka. start ts) |
+| `TRX_STARTED`|`time`| Human readable start time of the transaction |
+| `DIGEST`|`text`| The digest of the current executing SQL statement |
+| `ALL_SQLS` | `text` | A list of all executed SQL statements' digests |
+| `STATE`| `enum('Running', 'Lock waiting', 'Committing', 'RollingBack')`| The state of the transaction |
 | `WAITING_START_TIME` | `time` | The elapsed time since the start of the current lock waiting (if any) |
 | `SCOPE` | `enum('Global', 'Local')` | The scope of the transaction |
-| `ISOLATION_LEVEL` | `enum('RR', 'RC')` | |
+| `ISOLATION_LEVEL` | `enum('REPEATABLE-READ', 'READ-COMMITTED')` | |
 | `AUTOCOMMIT` | `bool` |  |
 | `SESSION_ID` | `unsigned bigint` | | 
 | `USER` | `varchar` | |
@@ -79,24 +79,28 @@ Several tables will be provided in `information_schema`. Some tables has both lo
 * Permission:
     * `PROCESS` privilege is needed to access this table.
 
-### Table `(CLUSTER_)DEAD_LOCK`
+### Table `(CLUSTER_)DEADLOCKS`
 
 | Field      | Type       | Comment |
 |------------|------------|---------|
 | `DEADLOCK_ID` | `int` | There needs multiple rows to represent information of a single deadlock event. This field is used to distinguish different events. |
 | `OCCUR_TIME`  | `time` | The physical time when the deadlock occurs |
+| `RETRYABLE` | `bool` | Is the deadlock retryable. TiDB tries to determine if the current statement is (indirectly) waiting for a lock locked by the current statement. |
 | `TRY_LOCK_TRX_ID` | `unsigned bigint` | The transaction ID (start ts) of the transaction that's trying to acquire the lock |
 | `CURRENT_SQL_DIGEST` | `text` | The SQL that's being blocked |
 | `KEY` | `varchar` | The key that's being locked, but locked by another transaction in the deadlock event |
-| `SQLS` | `text` | A list of the digest of SQL statements that the transaction has executed |
+| `ALL_SQLS` | `text` | A list of the digest of SQL statements that the transaction has executed |
 | `TRX_HOLDING_LOCK` | `unsigned bigint` | The transaction that's currently holding the lock. There will be another record in the table with the same `DEADLOCK_ID` for that transaction. |
 
 * Life span of rows:
     * Create after TiDB receive a deadlock error
     * FIFO，clean the oldest after buffer is full
 * Collecting, storing and querying:
-    * All of these information can be collected on TiDB side. It just need to add the information to the table when receives deadlock error from TiKV. The information of other transactions involved in the deadlock circle needed to be fetched from elsewhere (the `TIDB_TRX` table) when handling the deadlock error.
-    * Currently there are no much information in the deadlock error (it doesn't has the SQLs and keys' information),  which needs to be improved.
+    * All of these information can be collected on TiDB side. It just need to add the information to the table when receives deadlock error from TiKV. The information of other transactions involved in the deadlock circle needed to be fetched from elsewhere (the `CLUSTER_TIDB_TRX` table) when handling the deadlock error.
+    * TiKV needs to report more rich information in the deadlock error for collecting.
+    * There are two types of deadlock errors internally: retryable or non-retryable. The transaction will internally retry on retryable deadlocks and won't report error to the client. Therefore, the user are typically more interested in the non-retryable deadlocks. 
+        * Retryable deadlock errors are by default not collected, and can be enabled with configuration.
+        * Collecting `CLUSTER_TIDB_TRX` for more rich information for retryable deadlock is possible to make the performance worse. Whether it will be collected for retryable deadlock will be decided after some tests.
 * Permission:
   * `PROCESS` privilege is needed to access this table.
 
@@ -151,9 +155,25 @@ The locking key and `resource_group_tag` that comes from the `Context` of the pe
 
 The wait chain will be added to the `Deadlock` error which is returned by the `PessimisticLock` request, so that when deadlock happens, the full wait chain information can be passed to TiDB.
 
+### Configurations
+
+#### TiDB Config File `pessimistic-txn.tidb_deadlock_history_capacity`
+
+Specifies how many recent deadlock events each TiDB node should keep.
+Dynamically changeable via HTTP API.
+Value: 0 to 10000
+Default: 10
+
+#### TiDB Config File `pessimistic-txn.tidb_deadlock_history_collect_retryable`
+
+Specifies whether to collect retryable deadlock errors to the `(CLUSTER_)DEADLOCKS` table.
+Dynamically changeable via HTTP API.
+Value: 0 (do not collect) or 1 (collect)
+Default: 0
+
 ## Compatibility
 
-This feature is not expected to be incompatible with other features. During upgrading, when there are different versions of TiDB nodes exists at the same time, it's possible that the `CLUSTER_` prefixed tables may encounter errors. But since this feature is typically used by user manually, this shouldn't be a severe problem. So we don't need to care much about that.
+This feature is not expected to be incompatible with other features. During upgrading, when there are different versions of TiDB nodes exists at the same time, it's possible that the `CLUSTER_` prefixed tables may encounter errors. However, since this feature is typically used by user manually, this shouldn't be a severe problem. So we don't need to care much about that.
 
 ## Test Design
 
@@ -190,7 +210,7 @@ This feature is not expected to be incompatible with other features. During upgr
 
 * Since lock waiting on TiKV may timeout and retry, it's possible that in a single query to `DATA_LOCK_WAIT` table doesn't shows all (logical) lock waiting.
 * Information about internal transactions may not be collected in our first version of implementation.
-* Since TiDB need to query transaction information after it receives the deadlock error, the transactions' status may be changed during that time. As a result the information in `(CLUSTER_)DEAD_LOCK` table can't be promised to be accurate and complete.
+* Since TiDB need to query transaction information after it receives the deadlock error, the transactions' status may be changed during that time. As a result the information in `(CLUSTER_)DEADLOCKS` table can't be promised to be accurate and complete.
 * Statistics about transaction conflicts is still not enough.
 * Historical information of `TIDB_TRX` and `DATA_LOCK_WAITS` is not kept, which possibly makes it still difficult to investigate some kind of problems.
 * The SQL digest that's holding lock and blocking the current transaction is hard to retrieve and is not included in the current design.

domain/domain_test.go

@@ -287,7 +287,7 @@ func (*testSuite) TestT(c *C) {
 	c.Assert(dd, NotNil)
 	c.Assert(dd.GetLease(), Equals, 80*time.Millisecond)
 
-	snapTS := oracle.EncodeTSO(oracle.GetPhysical(time.Now()))
+	snapTS := oracle.GoTimeToTS(time.Now())
 	cs := &ast.CharsetOpt{
 		Chs: "utf8",
 		Col: "utf8_bin",
@@ -317,7 +317,7 @@ func (*testSuite) TestT(c *C) {
 	c.Assert(err, IsNil)
 
 	// for GetSnapshotInfoSchema
-	currSnapTS := oracle.EncodeTSO(oracle.GetPhysical(time.Now()))
+	currSnapTS := oracle.GoTimeToTS(time.Now())
 	currSnapIs, err := dom.GetSnapshotInfoSchema(currSnapTS)
 	c.Assert(err, IsNil)
 	c.Assert(currSnapIs, NotNil)

domain/infosync/info.go

@@ -559,7 +559,7 @@ func (is *InfoSyncer) ReportMinStartTS(store kv.Storage) {
 		logutil.BgLogger().Error("update minStartTS failed", zap.Error(err))
 		return
 	}
-	now := time.Unix(0, oracle.ExtractPhysical(currentVer.Ver)*1e6)
+	now := oracle.GetTimeFromTS(currentVer.Ver)
 	startTSLowerLimit := oracle.GoTimeToLowerLimitStartTS(now, tikv.MaxTxnTimeUse)
 
 	minStartTS := oracle.GoTimeToTS(now)

errno/errcode.go

@@ -855,6 +855,11 @@ const (
 	ErrGrantRole                                             = 3523
 	ErrRoleNotGranted                                        = 3530
 	ErrLockAcquireFailAndNoWaitSet                           = 3572
+	ErrCTERecursiveRequiresUnion                             = 3573
+	ErrCTERecursiveRequiresNonRecursiveFirst                 = 3574
+	ErrCTERecursiveForbidsAggregation                        = 3575
+	ErrCTERecursiveForbiddenJoinOrder                        = 3576
+	ErrInvalidRequiresSingleReference                        = 3577
 	ErrWindowNoSuchWindow                                    = 3579
 	ErrWindowCircularityInWindowGraph                        = 3580
 	ErrWindowNoChildPartitioning                             = 3581
@@ -877,6 +882,7 @@ const (
 	ErrWindowExplainJSON                                     = 3598
 	ErrWindowFunctionIgnoresFrame                            = 3599
 	ErrIllegalPrivilegeLevel                                 = 3619
+	ErrCTEMaxRecursionDepth                                  = 3636
 	ErrNotHintUpdatable                                      = 3637
 	ErrDataTruncatedFunctionalIndex                          = 3751
 	ErrDataOutOfRangeFunctionalIndex                         = 3752

errno/errname.go

@@ -355,7 +355,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
 	ErrViewSelectClause:                         mysql.Message("View's SELECT contains a '%s' clause", nil),
 	ErrViewSelectVariable:                       mysql.Message("View's SELECT contains a variable or parameter", nil),
 	ErrViewSelectTmptable:                       mysql.Message("View's SELECT refers to a temporary table '%-.192s'", nil),
-	ErrViewWrongList:                            mysql.Message("View's SELECT and view's field list have different column counts", nil),
+	ErrViewWrongList:                            mysql.Message("In definition of view, derived table or common table expression, SELECT list and column names list have different column counts", nil),
 	ErrWarnViewMerge:                            mysql.Message("View merge algorithm can't be used here for now (assumed undefined algorithm)", nil),
 	ErrWarnViewWithoutKey:                       mysql.Message("View being updated does not have complete key of underlying table in it", nil),
 	ErrViewInvalid:                              mysql.Message("View '%-.192s.%-.192s' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them", nil),
@@ -902,6 +902,12 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
 	ErrUnsupportedConstraintCheck:                            mysql.Message("%s is not supported", nil),
 	ErrDynamicPrivilegeNotRegistered:                         mysql.Message("Dynamic privilege '%s' is not registered with the server.", nil),
 	ErrIllegalPrivilegeLevel:                                 mysql.Message("Illegal privilege level specified for %s", nil),
+	ErrCTERecursiveRequiresUnion:                             mysql.Message("Recursive Common Table Expression '%s' should contain a UNION", nil),
+	ErrCTERecursiveRequiresNonRecursiveFirst:                 mysql.Message("Recursive Common Table Expression '%s' should have one or more non-recursive query blocks followed by one or more recursive ones", nil),
+	ErrCTERecursiveForbidsAggregation:                        mysql.Message("Recursive Common Table Expression '%s' can contain neither aggregation nor window functions in recursive query block", nil),
+	ErrCTERecursiveForbiddenJoinOrder:                        mysql.Message("In recursive query block of Recursive Common Table Expression '%s', the recursive table must neither be in the right argument of a LEFT JOIN, nor be forced to be non-first with join order hints", nil),
+	ErrInvalidRequiresSingleReference:                        mysql.Message("In recursive query block of Recursive Common Table Expression '%s', the recursive table must be referenced only once, and not in any subquery", nil),
+	ErrCTEMaxRecursionDepth:                                  mysql.Message("Recursive query aborted after %d iterations. Try increasing @@cte_max_recursion_depth to a larger value", nil),
 	// MariaDB errors.
 	ErrOnlyOneDefaultPartionAllowed:         mysql.Message("Only one DEFAULT partition allowed", nil),
 	ErrWrongPartitionTypeExpectedSystemTime: mysql.Message("Wrong partitioning type, expected type: `SYSTEM_TIME`", nil),

errors.toml

@@ -193,7 +193,7 @@ View's SELECT contains a '%s' clause
 
 ["ddl:1353"]
 error = '''
-View's SELECT and view's field list have different column counts
+In definition of view, derived table or common table expression, SELECT list and column names list have different column counts
 '''
 
 ["ddl:1481"]
@@ -561,6 +561,11 @@ error = '''
 Illegal privilege level specified for %s
 '''
 
+["executor:3636"]
+error = '''
+Recursive query aborted after %d iterations. Try increasing @@cte_max_recursion_depth to a larger value
+'''
+
 ["executor:3929"]
 error = '''
 Dynamic privilege '%s' is not registered with the server.
@@ -1016,6 +1021,31 @@ error = '''
 Unresolved name '%s' for %s hint
 '''
 
+["planner:3573"]
+error = '''
+Recursive Common Table Expression '%s' should contain a UNION
+'''
+
+["planner:3574"]
+error = '''
+Recursive Common Table Expression '%s' should have one or more non-recursive query blocks followed by one or more recursive ones
+'''
+
+["planner:3575"]
+error = '''
+Recursive Common Table Expression '%s' can contain neither aggregation nor window functions in recursive query block
+'''
+
+["planner:3576"]
+error = '''
+In recursive query block of Recursive Common Table Expression '%s', the recursive table must neither be in the right argument of a LEFT JOIN, nor be forced to be non-first with join order hints
+'''
+
+["planner:3577"]
+error = '''
+In recursive query block of Recursive Common Table Expression '%s', the recursive table must be referenced only once, and not in any subquery
+'''
+
 ["planner:3579"]
 error = '''
 Window name '%s' is not defined.

executor/batch_point_get_test.go

@@ -156,6 +156,16 @@ func (s *testBatchPointGetSuite) TestIssue18843(c *C) {
 	tk.MustQuery("select * from t18843 where f is null").Check(testkit.Rows("2 <nil>"))
 }
 
+func (s *testBatchPointGetSuite) TestIssue24562(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists ttt")
+	tk.MustExec("create table ttt(a enum(\"a\",\"b\",\"c\",\"d\"), primary key(a));")
+	tk.MustExec("insert into ttt values(1)")
+	tk.MustQuery("select * from ttt where ttt.a in (\"1\",\"b\")").Check(testkit.Rows())
+	tk.MustQuery("select * from ttt where ttt.a in (1,\"b\")").Check(testkit.Rows("a"))
+}
+
 func (s *testBatchPointGetSuite) TestBatchPointGetUnsignedHandleWithSort(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")

executor/builder.go

@@ -1534,7 +1534,9 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo
 			strings.ToLower(infoschema.TableClientErrorsSummaryByUser),
 			strings.ToLower(infoschema.TableClientErrorsSummaryByHost),
 			strings.ToLower(infoschema.TableTiDBTrx),
-			strings.ToLower(infoschema.ClusterTableTiDBTrx):
+			strings.ToLower(infoschema.ClusterTableTiDBTrx),
+			strings.ToLower(infoschema.TableDeadlocks),
+			strings.ToLower(infoschema.ClusterTableDeadlocks):
 			return &MemTableReaderExec{
 				baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()),
 				table:        v.Table,

executor/errors.go

@@ -50,8 +50,9 @@ var (
 	ErrIllegalPrivilegeLevel         = dbterror.ClassExecutor.NewStd(mysql.ErrIllegalPrivilegeLevel)
 	ErrInvalidSplitRegionRanges      = dbterror.ClassExecutor.NewStd(mysql.ErrInvalidSplitRegionRanges)
 
-	ErrBRIEBackupFailed  = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEBackupFailed)
-	ErrBRIERestoreFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIERestoreFailed)
-	ErrBRIEImportFailed  = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEImportFailed)
-	ErrBRIEExportFailed  = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEExportFailed)
+	ErrBRIEBackupFailed     = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEBackupFailed)
+	ErrBRIERestoreFailed    = dbterror.ClassExecutor.NewStd(mysql.ErrBRIERestoreFailed)
+	ErrBRIEImportFailed     = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEImportFailed)
+	ErrBRIEExportFailed     = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEExportFailed)
+	ErrCTEMaxRecursionDepth = dbterror.ClassExecutor.NewStd(mysql.ErrCTEMaxRecursionDepth)
 )

executor/executor.go

@@ -49,6 +49,7 @@ import (
 	"github.com/pingcap/tidb/sessionctx/stmtctx"
 	"github.com/pingcap/tidb/sessionctx/variable"
 	"github.com/pingcap/tidb/store/tikv"
+	tikverr "github.com/pingcap/tidb/store/tikv/error"
 	tikvstore "github.com/pingcap/tidb/store/tikv/kv"
 	tikvutil "github.com/pingcap/tidb/store/tikv/util"
 	"github.com/pingcap/tidb/table"
@@ -58,6 +59,7 @@ import (
 	"github.com/pingcap/tidb/util"
 	"github.com/pingcap/tidb/util/admin"
 	"github.com/pingcap/tidb/util/chunk"
+	"github.com/pingcap/tidb/util/deadlockhistory"
 	"github.com/pingcap/tidb/util/disk"
 	"github.com/pingcap/tidb/util/execdetails"
 	"github.com/pingcap/tidb/util/logutil"
@@ -983,6 +985,13 @@ func newLockCtx(seVars *variable.SessionVars, lockWaitTime int64) *tikvstore.Loc
 		LockKeysCount:         &seVars.StmtCtx.LockKeysCount,
 		LockExpired:           &seVars.TxnCtx.LockExpire,
 		ResourceGroupTag:      resourcegrouptag.EncodeResourceGroupTag(sqlDigest),
+		OnDeadlock: func(deadlock *tikverr.ErrDeadlock) {
+			// TODO: Support collecting retryable deadlocks according to the config.
+			if !deadlock.IsRetryable {
+				rec := deadlockhistory.ErrDeadlockToDeadlockRecord(deadlock)
+				deadlockhistory.GlobalDeadlockHistory.Push(rec)
+			}
+		},
 	}
 }