Merge branch 'master' of github.com:pingcap/tidb

Makefile

@@ -262,3 +262,10 @@ else
 	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' -cover github.com/pingcap/tidb/$(pkg) -check.p true -check.timeout 4s || { $(FAILPOINT_DISABLE); exit 1; }
 endif
 	@$(FAILPOINT_DISABLE)
+
+# Collect the daily benchmark data.
+# Usage:
+#	make bench-daily TO=/path/to/file.json
+bench-daily:
+	cd ./session && \
+	go test -run TestBenchDaily --date `git log -n1 --date=unix --pretty=format:%cd` --commit `git log -n1 --pretty=format:%h` --outfile $(TO)

config/config.go

@@ -654,7 +654,7 @@ var defaultConf = Config{
 	StmtSummary: StmtSummary{
 		Enable:              true,
 		EnableInternalQuery: false,
-		MaxStmtCount:        200,
+		MaxStmtCount:        3000,
 		MaxSQLLength:        4096,
 		RefreshInterval:     1800,
 		HistorySize:         24,

config/config.toml.example

@@ -454,7 +454,7 @@ enable = true
 enable-internal-query = false
 
 # max number of statements kept in memory.
-max-stmt-count = 200
+max-stmt-count = 3000
 
 # max length of displayed normalized sql and sample sql.
 max-sql-length = 4096

domain/domain.go

@@ -622,10 +622,13 @@ func (do *Domain) Close() {
 		terror.Log(errors.Trace(do.etcdClient.Close()))
 	}
 
-	do.sysSessionPool.Close()
 	do.slowQuery.Close()
+
 	do.cancel()
 	do.wg.Wait()
+
+	do.sysSessionPool.Close()
+
 	logutil.BgLogger().Info("domain closed", zap.Duration("take time", time.Since(startTime)))
 }
 

errors.toml

@@ -501,6 +501,11 @@ error = '''
 %-.128s command denied to user '%-.48s'@'%-.64s' for table '%-.64s'
 '''
 
+["executor:1144"]
+error = '''
+Illegal GRANT/REVOKE command; please consult the manual to see which privileges can be used
+'''
+
 ["executor:1213"]
 error = '''
 Deadlock found when trying to get lock; try restarting transaction

executor/errors.go

@@ -30,6 +30,7 @@ var (
 	ErrBatchInsertFail      = dbterror.ClassExecutor.NewStd(mysql.ErrBatchInsertFail)
 	ErrUnsupportedPs        = dbterror.ClassExecutor.NewStd(mysql.ErrUnsupportedPs)
 	ErrSubqueryMoreThan1Row = dbterror.ClassExecutor.NewStd(mysql.ErrSubqueryNo1Row)
+	ErrIllegalGrantForTable = dbterror.ClassExecutor.NewStd(mysql.ErrIllegalGrantForTable)
 
 	ErrCantCreateUserWithGrant       = dbterror.ClassExecutor.NewStd(mysql.ErrCantCreateUserWithGrant)
 	ErrPasswordNoMatch               = dbterror.ClassExecutor.NewStd(mysql.ErrPasswordNoMatch)

executor/executor_test.go

@@ -5733,8 +5733,8 @@ func (s *testRecoverTable) SetUpSuite(c *C) {
 }
 
 func (s *testRecoverTable) TearDownSuite(c *C) {
-	s.store.Close()
 	s.dom.Close()
+	s.store.Close()
 }
 
 func (s *testRecoverTable) mockGC(tk *testkit.TestKit) (string, string, string, func()) {

executor/grant.go

@@ -547,131 +547,98 @@ func (e *GrantExec) grantColumnLevel(priv *ast.PrivElem, user *ast.UserSpec, int
 // composeGlobalPrivUpdate composes update stmt assignment list string for global scope privilege update.
 func composeGlobalPrivUpdate(sql *strings.Builder, priv mysql.PrivilegeType, value string) error {
 	if priv != mysql.AllPriv {
-		col, ok := mysql.Priv2UserCol[priv]
-		if !ok {
-			return errors.Errorf("Unknown priv: %v", priv)
+		if priv != mysql.GrantPriv && !mysql.AllGlobalPrivs.Has(priv) {
+			return ErrWrongUsage.GenWithStackByArgs("GLOBAL GRANT", "NON-GLOBAL PRIVILEGES")
 		}
-		sqlexec.MustFormatSQL(sql, "%n=%?", col, value)
+		sqlexec.MustFormatSQL(sql, "%n=%?", priv.ColumnString(), value)
 		return nil
 	}
 
 	for i, v := range mysql.AllGlobalPrivs {
 		if i > 0 {
 			sqlexec.MustFormatSQL(sql, ",")
 		}
-
-		k, ok := mysql.Priv2UserCol[v]
-		if !ok {
-			return errors.Errorf("Unknown priv %v", priv)
-		}
-
-		sqlexec.MustFormatSQL(sql, "%n=%?", k, value)
+		sqlexec.MustFormatSQL(sql, "%n=%?", v.ColumnString(), value)
 	}
 	return nil
 }
 
 // composeDBPrivUpdate composes update stmt assignment list for db scope privilege update.
 func composeDBPrivUpdate(sql *strings.Builder, priv mysql.PrivilegeType, value string) error {
 	if priv != mysql.AllPriv {
-		col, ok := mysql.Priv2UserCol[priv]
-		if !ok {
-			return errors.Errorf("Unknown priv: %v", priv)
+		if priv != mysql.GrantPriv && !mysql.AllDBPrivs.Has(priv) {
+			return ErrWrongUsage.GenWithStackByArgs("DB GRANT", "NON-DB PRIVILEGES")
 		}
-		sqlexec.MustFormatSQL(sql, "%n=%?", col, value)
+		sqlexec.MustFormatSQL(sql, "%n=%?", priv.ColumnString(), value)
 		return nil
 	}
 
 	for i, p := range mysql.AllDBPrivs {
 		if i > 0 {
 			sqlexec.MustFormatSQL(sql, ",")
 		}
-
-		v, ok := mysql.Priv2UserCol[p]
-		if !ok {
-			return errors.Errorf("Unknown priv %v", priv)
-		}
-
-		sqlexec.MustFormatSQL(sql, "%n=%?", v, value)
+		sqlexec.MustFormatSQL(sql, "%n=%?", p.ColumnString(), value)
 	}
 	return nil
 }
 
-func privUpdateForGrant(cur []string, priv mysql.PrivilegeType) ([]string, error) {
-	p, ok := mysql.Priv2SetStr[priv]
-	if !ok {
-		return nil, errors.Errorf("Unknown priv: %v", priv)
-	}
-	cur = addToSet(cur, p)
-	return cur, nil
-}
-
 // composeTablePrivUpdateForGrant composes update stmt assignment list for table scope privilege update.
 func composeTablePrivUpdateForGrant(ctx sessionctx.Context, sql *strings.Builder, priv mysql.PrivilegeType, name string, host string, db string, tbl string) error {
 	var newTablePriv, newColumnPriv []string
-	var tblPrivs, colPrivs []mysql.PrivilegeType
 	if priv != mysql.AllPriv {
+		// TODO: https://github.com/pingcap/parser/pull/581 removed privs from all priv lists
+		// it is to avoid add GRANT in GRANT ALL SQLs
+		// WithGRANT seems broken, fix it later
+		if priv != mysql.GrantPriv && !mysql.AllTablePrivs.Has(priv) {
+			return ErrIllegalGrantForTable
+		}
+
 		currTablePriv, currColumnPriv, err := getTablePriv(ctx, name, host, db, tbl)
 		if err != nil {
 			return err
 		}
-		newTablePriv = setFromString(currTablePriv)
-		newColumnPriv = setFromString(currColumnPriv)
-		tblPrivs = []mysql.PrivilegeType{priv}
-		for _, cp := range mysql.AllColumnPrivs {
-			// in case it is not a column priv
-			if cp == priv {
-				colPrivs = []mysql.PrivilegeType{priv}
-				break
-			}
+		newTablePriv = SetFromString(currTablePriv)
+		newTablePriv = addToSet(newTablePriv, priv.SetString())
+
+		newColumnPriv = SetFromString(currColumnPriv)
+		if mysql.AllColumnPrivs.Has(priv) {
+			newColumnPriv = addToSet(newColumnPriv, priv.SetString())
 		}
 	} else {
-		tblPrivs = mysql.AllTablePrivs
-		colPrivs = mysql.AllColumnPrivs
-	}
-
-	var err error
-	for _, p := range tblPrivs {
-		newTablePriv, err = privUpdateForGrant(newTablePriv, p)
-		if err != nil {
-			return err
+		for _, p := range mysql.AllTablePrivs {
+			newTablePriv = addToSet(newTablePriv, p.SetString())
 		}
-	}
 
-	for _, p := range colPrivs {
-		newColumnPriv, err = privUpdateForGrant(newColumnPriv, p)
-		if err != nil {
-			return err
+		for _, p := range mysql.AllColumnPrivs {
+			newColumnPriv = addToSet(newColumnPriv, p.SetString())
 		}
 	}
 
-	sqlexec.MustFormatSQL(sql, `Table_priv=%?, Column_priv=%?, Grantor=%?`, strings.Join(newTablePriv, ","), strings.Join(newColumnPriv, ","), ctx.GetSessionVars().User.String())
+	sqlexec.MustFormatSQL(sql, `Table_priv=%?, Column_priv=%?, Grantor=%?`, setToString(newTablePriv), setToString(newColumnPriv), ctx.GetSessionVars().User.String())
 	return nil
 }
 
 // composeColumnPrivUpdateForGrant composes update stmt assignment list for column scope privilege update.
 func composeColumnPrivUpdateForGrant(ctx sessionctx.Context, sql *strings.Builder, priv mysql.PrivilegeType, name string, host string, db string, tbl string, col string) error {
 	var newColumnPriv []string
-	var colPrivs []mysql.PrivilegeType
 	if priv != mysql.AllPriv {
+		if !mysql.AllColumnPrivs.Has(priv) {
+			return ErrWrongUsage.GenWithStackByArgs("COLUMN GRANT", "NON-COLUMN PRIVILEGES")
+		}
+
 		currColumnPriv, err := getColumnPriv(ctx, name, host, db, tbl, col)
 		if err != nil {
 			return err
 		}
-		newColumnPriv = setFromString(currColumnPriv)
-		colPrivs = []mysql.PrivilegeType{priv}
+		newColumnPriv = SetFromString(currColumnPriv)
+		newColumnPriv = addToSet(newColumnPriv, priv.SetString())
 	} else {
-		colPrivs = mysql.AllColumnPrivs
-	}
-
-	var err error
-	for _, p := range colPrivs {
-		newColumnPriv, err = privUpdateForGrant(newColumnPriv, p)
-		if err != nil {
-			return err
+		for _, p := range mysql.AllColumnPrivs {
+			newColumnPriv = addToSet(newColumnPriv, p.SetString())
 		}
 	}
 
-	sqlexec.MustFormatSQL(sql, `Column_priv=%?`, strings.Join(newColumnPriv, ","))
+	sqlexec.MustFormatSQL(sql, `Column_priv=%?`, setToString(newColumnPriv))
 	return nil
 }
 

executor/grant_test.go

@@ -93,6 +93,9 @@ func (s *testSuite3) TestGrantDBScope(c *C) {
 	// Grant in wrong scope.
 	_, err := tk.Exec(` grant create user on test.* to 'testDB1'@'localhost';`)
 	c.Assert(terror.ErrorEqual(err, executor.ErrWrongUsage.GenWithStackByArgs("DB GRANT", "GLOBAL PRIVILEGES")), IsTrue)
+
+	_, err = tk.Exec("GRANT SUPER ON test.* TO 'testDB1'@'localhost';")
+	c.Assert(terror.ErrorEqual(err, executor.ErrWrongUsage.GenWithStackByArgs("DB GRANT", "NON-DB PRIVILEGES")), IsTrue)
 }
 
 func (s *testSuite3) TestWithGrantOption(c *C) {
@@ -116,7 +119,7 @@ func (s *testSuite3) TestWithGrantOption(c *C) {
 	tk.MustQuery("SELECT grant_priv FROM mysql.user WHERE User=\"testWithGrant1\"").Check(testkit.Rows("Y"))
 }
 
-func (s *testSuiteP1) TestTableScope(c *C) {
+func (s *testSuiteP1) TestGrantTableScope(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	// Create a new user.
 	createUserSQL := `CREATE USER 'testTbl'@'localhost' IDENTIFIED BY '123';`
@@ -152,9 +155,12 @@ func (s *testSuiteP1) TestTableScope(c *C) {
 		p := fmt.Sprintf("%v", row[0])
 		c.Assert(strings.Index(p, mysql.Priv2SetStr[v]), Greater, -1)
 	}
+
+	_, err := tk.Exec("GRANT SUPER ON test2 TO 'testTbl1'@'localhost';")
+	c.Assert(err, ErrorMatches, "\\[executor:1144\\]Illegal GRANT/REVOKE command; please consult the manual to see which privileges can be used")
 }
 
-func (s *testSuite3) TestColumnScope(c *C) {
+func (s *testSuite3) TestGrantColumnScope(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	// Create a new user.
 	createUserSQL := `CREATE USER 'testCol'@'localhost' IDENTIFIED BY '123';`
@@ -192,6 +198,9 @@ func (s *testSuite3) TestColumnScope(c *C) {
 		p := fmt.Sprintf("%v", row[0])
 		c.Assert(strings.Index(p, mysql.Priv2SetStr[v]), Greater, -1)
 	}
+
+	_, err := tk.Exec("GRANT SUPER(c2) ON test3 TO 'testCol1'@'localhost';")
+	c.Assert(err, ErrorMatches, "\\[executor:1221\\]Incorrect usage of COLUMN GRANT and NON-COLUMN PRIVILEGES")
 }
 
 func (s *testSuite3) TestIssue2456(c *C) {

executor/index_lookup_hash_join.go

@@ -598,10 +598,10 @@ func (iw *indexHashJoinInnerWorker) handleTask(ctx context.Context, task *indexH
 	// TODO(XuHuaiyu): we may always use the smaller side to build the hashtable.
 	go util.WithRecovery(func() { iw.buildHashTableForOuterResult(ctx, task, h) }, iw.handleHashJoinInnerWorkerPanic)
 	err := iw.fetchInnerResults(ctx, task.lookUpJoinTask)
+	iw.wg.Wait()
 	if err != nil {
 		return err
 	}
-	iw.wg.Wait()
 
 	joinStartTime = time.Now()
 	if !task.keepOuterOrder {

executor/revoke.go

@@ -303,13 +303,13 @@ func composeTablePrivUpdateForRevoke(ctx sessionctx.Context, sql *strings.Builde
 			return err
 		}
 
-		newTablePriv = setFromString(currTablePriv)
+		newTablePriv = SetFromString(currTablePriv)
 		newTablePriv, err = privUpdateForRevoke(newTablePriv, priv)
 		if err != nil {
 			return err
 		}
 
-		newColumnPriv = setFromString(currColumnPriv)
+		newColumnPriv = SetFromString(currColumnPriv)
 		newColumnPriv, err = privUpdateForRevoke(newColumnPriv, priv)
 		if err != nil {
 			return err
@@ -329,7 +329,7 @@ func composeColumnPrivUpdateForRevoke(ctx sessionctx.Context, sql *strings.Build
 			return err
 		}
 
-		newColumnPriv = setFromString(currColumnPriv)
+		newColumnPriv = SetFromString(currColumnPriv)
 		newColumnPriv, err = privUpdateForRevoke(newColumnPriv, priv)
 		if err != nil {
 			return err

executor/revoke_test.go

@@ -94,15 +94,16 @@ func (s *testSuite1) TestRevokeTableScope(c *C) {
 		c.Assert(rows, HasLen, 1)
 		row := rows[0]
 		c.Assert(row, HasLen, 1)
-		op := mysql.Priv2SetStr[v]
+
+		op := v.SetString()
 		found := false
-		for _, v := range strings.Split(fmt.Sprintf("%v", row[0]), ",") {
-			if v == op {
+		for _, p := range executor.SetFromString(fmt.Sprintf("%s", row[0])) {
+			if op == p {
 				found = true
 				break
 			}
 		}
-		c.Assert(found, IsFalse)
+		c.Assert(found, IsFalse, Commentf("%s", mysql.Priv2SetStr[v]))
 	}
 
 	// Revoke all table scope privs.

executor/show_test.go

@@ -1130,7 +1130,7 @@ func (s *testSuite5) TestShowBuiltin(c *C) {
 	res := tk.MustQuery("show builtins;")
 	c.Assert(res, NotNil)
 	rows := res.Rows()
-	const builtinFuncNum = 269
+	const builtinFuncNum = 271
 	c.Assert(builtinFuncNum, Equals, len(rows))
 	c.Assert("abs", Equals, rows[0][0].(string))
 	c.Assert("yearweek", Equals, rows[builtinFuncNum-1][0].(string))