server: support resigning ddl owner, use http method ddl/owner/resign

docs/tidb_http_api.md

@@ -132,3 +132,11 @@ timezone.*
     curl http://{TiDBIP}:10080/tables/{colID}/{colFlag}/{colLen}?rowBin={val}
     ```
     *Hint: For the column which field type is timezone dependent, e.g. `timestamp`, convert its value to UTC timezone.*
+
+1. Resign the ddl owner, let tidb start a new ddl owner election.
+
+    ```shell
+    curl http://{TiDBIP}:10080/ddl/owner/resign
+    ```
+
+    **Note**: If you request a tidb that is not ddl owner, the response will be `This node is not a ddl owner, can't be resigned.`

owner/manager.go

@@ -18,6 +18,7 @@ import (
 	"math"
 	"os"
 	"strconv"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -51,6 +52,8 @@ type Manager interface {
 	GetOwnerID(ctx context.Context) (string, error)
 	// CampaignOwner campaigns the owner.
 	CampaignOwner(ctx context.Context) error
+	// ResignOwner lets the owner start a new election.
+	ResignOwner(ctx context.Context) error
 	// Cancel cancels this etcd ownerManager campaign.
 	Cancel()
 }
@@ -70,22 +73,26 @@ type DDLOwnerChecker interface {
 
 // ownerManager represents the structure which is used for electing owner.
 type ownerManager struct {
-	owner   int32
-	id      string // id is the ID of the manager.
-	key     string
-	prompt  string
-	etcdCli *clientv3.Client
-	cancel  context.CancelFunc
+	owner     int32
+	id        string // id is the ID of the manager.
+	key       string
+	prompt    string
+	logPrefix string
+	etcdCli   *clientv3.Client
+	cancel    context.CancelFunc
+	elec      *concurrency.Election
+	mu        sync.Mutex
 }
 
 // NewOwnerManager creates a new Manager.
 func NewOwnerManager(etcdCli *clientv3.Client, prompt, id, key string, cancel context.CancelFunc) Manager {
 	return &ownerManager{
-		etcdCli: etcdCli,
-		id:      id,
-		key:     key,
-		prompt:  prompt,
-		cancel:  cancel,
+		etcdCli:   etcdCli,
+		id:        id,
+		key:       key,
+		prompt:    prompt,
+		cancel:    cancel,
+		logPrefix: fmt.Sprintf("[%s] %s ownerManager %s", prompt, key, id),
 	}
 }
 
@@ -179,6 +186,37 @@ func (m *ownerManager) CampaignOwner(ctx context.Context) error {
 	return nil
 }
 
+// ResignOwner lets the owner start a new election.
+func (m *ownerManager) ResignOwner(ctx context.Context) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if !m.IsOwner() || m.elec == nil {
+		return errors.Errorf("This node is not a ddl owner, can't be resigned.")
+	}
+
+	err := m.elec.Resign(ctx)
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	log.Warnf("%s Resign ddl owner success!", m.logPrefix)
+	return nil
+}
+
+func (m *ownerManager) toBeOwner(elec *concurrency.Election) {
+	m.mu.Lock()
+	m.elec = elec
+	m.SetOwner(true)
+	m.mu.Unlock()
+}
+
+func (m *ownerManager) retireOwner() {
+	m.mu.Lock()
+	m.SetOwner(false)
+	m.elec = nil
+	m.mu.Unlock()
+}
+
 func (m *ownerManager) campaignLoop(ctx context.Context, etcdSession *concurrency.Session) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -188,7 +226,7 @@ func (m *ownerManager) campaignLoop(ctx context.Context, etcdSession *concurrenc
 		}
 	}()
 
-	logPrefix := fmt.Sprintf("[%s] %s ownerManager %s", m.prompt, m.key, m.id)
+	logPrefix := m.logPrefix
 	var err error
 	for {
 		if err != nil {
@@ -232,9 +270,10 @@ func (m *ownerManager) campaignLoop(ctx context.Context, etcdSession *concurrenc
 		if err != nil {
 			continue
 		}
-		m.SetOwner(true)
+
+		m.toBeOwner(elec)
 		m.watchOwner(ctx, etcdSession, ownerKey)
-		m.SetOwner(false)
+		m.retireOwner()
 
 		metrics.CampaignOwnerCounter.WithLabelValues(m.prompt, metrics.NoLongerOwner).Inc()
 		log.Warnf("%s isn't the owner", logPrefix)

owner/mock.go

@@ -76,3 +76,11 @@ func (m *mockManager) CampaignOwner(_ context.Context) error {
 	m.SetOwner(true)
 	return nil
 }
+
+// ResignOwner lets the owner start a new election.
+func (m *mockManager) ResignOwner(ctx context.Context) error {
+	if m.IsOwner() {
+		m.SetOwner(false)
+	}
+	return nil
+}

server/http_handler.go

@@ -303,6 +303,25 @@ func (t *tikvHandlerTool) getAllHistoryDDL() ([]*model.Job, error) {
 	return jobs, nil
 }
 
+func (t *tikvHandlerTool) resignDDLOwner() error {
+	s, err := session.CreateSession(t.store.(kv.Storage))
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	if s != nil {
+		defer s.Close()
+	}
+
+	dom := domain.GetDomain(s.(sessionctx.Context))
+	ownerMgr := dom.DDL().OwnerManager()
+	err = ownerMgr.ResignOwner(context.Background())
+	if err != nil {
+		return errors.Trace(err)
+	}
+	return nil
+}
+
 // settingsHandler is the handler for list tidb server settings.
 type settingsHandler struct {
 }
@@ -334,6 +353,11 @@ type ddlHistoryJobHandler struct {
 	*tikvHandlerTool
 }
 
+// ddlResignOwnerHandler is the handler for resigning ddl owner.
+type ddlResignOwnerHandler struct {
+	*tikvHandlerTool
+}
+
 type serverInfoHandler struct {
 	*tikvHandlerTool
 }
@@ -713,6 +737,18 @@ func (h ddlHistoryJobHandler) ServeHTTP(w http.ResponseWriter, req *http.Request
 	return
 }
 
+// ServeHTTP handles request of resigning ddl owner.
+func (h ddlResignOwnerHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	err := h.resignDDLOwner()
+	if err != nil {
+		log.Error(err)
+		writeError(w, err)
+		return
+	}
+
+	writeData(w, "success!")
+}
+
 func (h tableHandler) getPDAddr() ([]string, error) {
 	var pdAddrs []string
 	etcd, ok := h.store.(domain.EtcdBackend)

server/http_status.go

@@ -52,6 +52,7 @@ func (s *Server) startHTTPServer() {
 	router.Handle("/schema/{db}/{table}", schemaHandler{tikvHandlerTool})
 	router.Handle("/tables/{colID}/{colTp}/{colFlag}/{colLen}", valueHandler{})
 	router.Handle("/ddl/history", ddlHistoryJobHandler{tikvHandlerTool})
+	router.Handle("/ddl/owner/resign", ddlResignOwnerHandler{tikvHandlerTool})
 
 	// HTTP path for get server info.
 	router.Handle("/info", serverInfoHandler{tikvHandlerTool})