meta: make auto increment id can be adjust. (#10978)

ddl/db_test.go

@@ -27,6 +27,7 @@ import (
 
 	. "github.com/pingcap/check"
 	"github.com/pingcap/errors"
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/mysql"
 	tmysql "github.com/pingcap/parser/mysql"
@@ -2163,7 +2164,11 @@ func (s *testDBSuite4) TestComment(c *C) {
 	s.tk.MustExec("drop table if exists ct, ct1")
 }
 
-func (s *testDBSuite5) TestRebaseAutoID(c *C) {
+func (s *testDBSuite4) TestRebaseAutoID(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
 	s.tk = testkit.NewTestKit(c, s.store)
 	s.tk.MustExec("use " + s.schemaName)
 
@@ -2758,7 +2763,12 @@ func (s *testDBSuite1) TestModifyColumnCharset(c *C) {
 
 }
 
-func (s *testDBSuite2) TestAlterShardRowIDBits(c *C) {
+func (s *testDBSuite4) TestAlterShardRowIDBits(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
+
 	s.tk = testkit.NewTestKit(c, s.store)
 	tk := s.tk
 

ddl/serial_test.go

@@ -241,6 +241,10 @@ func (s *testSerialSuite) TestRecoverTableByJobID(c *C) {
 }
 
 func (s *testSerialSuite) TestRecoverTableByTableName(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("create database if not exists test_recover")
 	tk.MustExec("use test_recover")

executor/ddl_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	. "github.com/pingcap/check"
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
@@ -343,6 +344,10 @@ func (s *testSuite3) TestDefaultDBAfterDropCurDB(c *C) {
 }
 
 func (s *testSuite3) TestRenameTable(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
 	tk := testkit.NewTestKit(c, s.store)
 
 	tk.MustExec("create database rename1")

executor/seqtest/seq_executor_test.go

@@ -708,6 +708,10 @@ func checkGoroutineExists(keyword string) bool {
 }
 
 func (s *seqTestSuite) TestAdminShowNextID(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
 	step := int64(10)
 	autoIDStep := autoid.GetStep()
 	autoid.SetStep(step)

meta/autoid/autoid.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/cznic/mathutil"
 	"github.com/pingcap/errors"
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/parser/terror"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/meta"
@@ -30,6 +31,12 @@ import (
 	"go.uber.org/zap"
 )
 
+const (
+	minStep            = 1000
+	maxStep            = 2000000
+	defaultConsumeTime = 10 * time.Second
+)
+
 // Test needs to change it, so it's a variable.
 var step = int64(30000)
 
@@ -59,8 +66,10 @@ type allocator struct {
 	end   int64
 	store kv.Storage
 	// dbID is current database's ID.
-	dbID       int64
-	isUnsigned bool
+	dbID          int64
+	isUnsigned    bool
+	lastAllocTime time.Time
+	step          int64
 }
 
 // GetStep is only used by tests
@@ -124,7 +133,7 @@ func (alloc *allocator) rebase4Unsigned(tableID int64, requiredBase uint64, allo
 		uCurrentEnd := uint64(currentEnd)
 		if allocIDs {
 			newBase = mathutil.MaxUint64(uCurrentEnd, requiredBase)
-			newEnd = mathutil.MinUint64(math.MaxUint64-uint64(step), newBase) + uint64(step)
+			newEnd = mathutil.MinUint64(math.MaxUint64-uint64(alloc.step), newBase) + uint64(alloc.step)
 		} else {
 			if uCurrentEnd >= requiredBase {
 				newBase = uCurrentEnd
@@ -169,7 +178,7 @@ func (alloc *allocator) rebase4Signed(tableID, requiredBase int64, allocIDs bool
 		}
 		if allocIDs {
 			newBase = mathutil.MaxInt64(currentEnd, requiredBase)
-			newEnd = mathutil.MinInt64(math.MaxInt64-step, newBase) + step
+			newEnd = mathutil.MinInt64(math.MaxInt64-alloc.step, newBase) + alloc.step
 		} else {
 			if currentEnd >= requiredBase {
 				newBase = currentEnd
@@ -215,21 +224,24 @@ func (alloc *allocator) alloc4Unsigned(tableID int64) (int64, error) {
 	if alloc.base == alloc.end { // step
 		var newBase, newEnd int64
 		startTime := time.Now()
+		consumeDur := startTime.Sub(alloc.lastAllocTime)
+		alloc.step = NextStep(alloc.step, consumeDur)
 		err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error {
 			m := meta.NewMeta(txn)
 			var err1 error
 			newBase, err1 = m.GetAutoTableID(alloc.dbID, tableID)
 			if err1 != nil {
 				return err1
 			}
-			tmpStep := int64(mathutil.MinUint64(math.MaxUint64-uint64(newBase), uint64(step)))
+			tmpStep := int64(mathutil.MinUint64(math.MaxUint64-uint64(newBase), uint64(alloc.step)))
 			newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep)
 			return err1
 		})
 		metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDAlloc, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
 		if err != nil {
 			return 0, err
 		}
+		alloc.lastAllocTime = time.Now()
 		if uint64(newBase) == math.MaxUint64 {
 			return 0, ErrAutoincReadFailed
 		}
@@ -248,21 +260,24 @@ func (alloc *allocator) alloc4Signed(tableID int64) (int64, error) {
 	if alloc.base == alloc.end { // step
 		var newBase, newEnd int64
 		startTime := time.Now()
+		consumeDur := startTime.Sub(alloc.lastAllocTime)
+		alloc.step = NextStep(alloc.step, consumeDur)
 		err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error {
 			m := meta.NewMeta(txn)
 			var err1 error
 			newBase, err1 = m.GetAutoTableID(alloc.dbID, tableID)
 			if err1 != nil {
 				return err1
 			}
-			tmpStep := mathutil.MinInt64(math.MaxInt64-newBase, step)
+			tmpStep := mathutil.MinInt64(math.MaxInt64-newBase, alloc.step)
 			newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep)
 			return err1
 		})
 		metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDAlloc, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
 		if err != nil {
 			return 0, err
 		}
+		alloc.lastAllocTime = time.Now()
 		if newBase == math.MaxInt64 {
 			return 0, ErrAutoincReadFailed
 		}
@@ -290,12 +305,32 @@ func (alloc *allocator) Alloc(tableID int64) (int64, error) {
 	return alloc.alloc4Signed(tableID)
 }
 
+// NextStep return new auto id step according to previous step and consuming time.
+func NextStep(curStep int64, consumeDur time.Duration) int64 {
+	failpoint.Inject("mockAutoIDChange", func(val failpoint.Value) {
+		if val.(bool) {
+			failpoint.Return(step)
+		}
+	})
+
+	consumeRate := defaultConsumeTime.Seconds() / consumeDur.Seconds()
+	res := int64(float64(curStep) * consumeRate)
+	if res < minStep {
+		return minStep
+	} else if res > maxStep {
+		return maxStep
+	}
+	return res
+}
+
 // NewAllocator returns a new auto increment id generator on the store.
 func NewAllocator(store kv.Storage, dbID int64, isUnsigned bool) Allocator {
 	return &allocator{
-		store:      store,
-		dbID:       dbID,
-		isUnsigned: isUnsigned,
+		store:         store,
+		dbID:          dbID,
+		isUnsigned:    isUnsigned,
+		step:          step,
+		lastAllocTime: time.Now(),
 	}
 }
 

meta/autoid/autoid_test.go

@@ -21,6 +21,7 @@ import (
 
 	. "github.com/pingcap/check"
 	"github.com/pingcap/errors"
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/meta"
@@ -39,6 +40,11 @@ type testSuite struct {
 }
 
 func (*testSuite) TestT(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
+
 	store, err := mockstore.NewMockTikvStore()
 	c.Assert(err, IsNil)
 	defer store.Close()
@@ -130,6 +136,11 @@ func (*testSuite) TestT(c *C) {
 }
 
 func (*testSuite) TestUnsignedAutoid(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
+
 	store, err := mockstore.NewMockTikvStore()
 	c.Assert(err, IsNil)
 	defer store.Close()
@@ -315,3 +326,13 @@ func (*testSuite) TestRollbackAlloc(c *C) {
 	c.Assert(alloc.Base(), Equals, int64(0))
 	c.Assert(alloc.End(), Equals, int64(0))
 }
+
+// TestNextStep tests generate next auto id step.
+func (*testSuite) TestNextStep(c *C) {
+	nextStep := autoid.NextStep(2000000, 1*time.Nanosecond)
+	c.Assert(nextStep, Equals, int64(2000000))
+	nextStep = autoid.NextStep(678910, 10*time.Second)
+	c.Assert(nextStep, Equals, int64(678910))
+	nextStep = autoid.NextStep(50000, 10*time.Minute)
+	c.Assert(nextStep, Equals, int64(1000))
+}