server: add /debug/zip api to get useful debuginfo at once (#9651)

docs/tidb_http_api.md

@@ -156,3 +156,16 @@ timezone.*
     ```
 
     **Note**: If you request a tidb that is not ddl owner, the response will be `This node is not a ddl owner, can't be resigned.`
+
+1. Download TiDB debug info
+
+     ```shell
+    curl http://{TiDBIP}:10080/debug/zip
+    ```
+    zip file will include:
+
+    - Go heap pprof(after GC)
+    - Go cpu pprof(10s)
+    - Go mutex pprof
+    - Full goroutine
+    - TiDB config and version

go.mod

@@ -44,7 +44,7 @@ require (
 	github.com/opentracing/basictracer-go v1.0.0
 	github.com/opentracing/opentracing-go v1.0.2
 	github.com/pingcap/check v0.0.0-20171206051426-1c287c953996
-	github.com/pingcap/errors v0.11.0
+	github.com/pingcap/errors v0.11.1
 	github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3
 	github.com/pingcap/goleveldb v0.0.0-20171020084629-8d44bfdf1030
 	github.com/pingcap/kvproto v0.0.0-20190226063853-f6c0b7ffff11

go.sum

@@ -84,8 +84,8 @@ github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 h1:ZBdiJCMan6GSo/aPAM7gywcUKa0z58gczVrnG6TQnAQ=
 github.com/pingcap/check v0.0.0-20171206051426-1c287c953996/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
-github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4=
-github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pingcap/errors v0.11.1 h1:BXFZ6MdDd2U1uJUa2sRAWTmm+nieEzuyYM0R4aUTcC8=
+github.com/pingcap/errors v0.11.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3 h1:04yuCf5NMvLU8rB2m4Qs3rynH7EYpMno3lHkewIOdMo=
 github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns=
 github.com/pingcap/goleveldb v0.0.0-20171020084629-8d44bfdf1030 h1:XJLuW0lsP7vAtQ2iPjZwvXZ14m5urp9No+Qr06ZZcTo=

server/http_status.go

@@ -14,15 +14,21 @@
 package server
 
 import (
+	"archive/zip"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/pprof"
+	"runtime"
+	rpprof "runtime/pprof"
+	"strconv"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
+	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/util/printer"
 	"github.com/prometheus/client_golang/prometheus"
@@ -88,6 +94,104 @@ func (s *Server) startHTTPServer() {
 	serverMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
 	serverMux.HandleFunc("/debug/pprof/trace", pprof.Trace)
 
+	serveError := func(w http.ResponseWriter, status int, txt string) {
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.Header().Set("X-Go-Pprof", "1")
+		w.Header().Del("Content-Disposition")
+		w.WriteHeader(status)
+		_, err := fmt.Fprintln(w, txt)
+		terror.Log(err)
+	}
+
+	sleep := func(w http.ResponseWriter, d time.Duration) {
+		var clientGone <-chan bool
+		if cn, ok := w.(http.CloseNotifier); ok {
+			clientGone = cn.CloseNotify()
+		}
+		select {
+		case <-time.After(d):
+		case <-clientGone:
+		}
+	}
+
+	serverMux.HandleFunc("/debug/zip", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="tidb_debug"`+time.Now().Format("20060102150405")+".zip"))
+
+		// dump goroutine/heap/mutex
+		items := []struct {
+			name   string
+			gc     int
+			debug  int
+			second int
+		}{
+			{name: "goroutine", debug: 2},
+			{name: "heap", gc: 1},
+			{name: "mutex"},
+		}
+		zw := zip.NewWriter(w)
+		for _, item := range items {
+			p := rpprof.Lookup(item.name)
+			if p == nil {
+				serveError(w, http.StatusNotFound, "Unknown profile")
+				return
+			}
+			if item.gc > 0 {
+				runtime.GC()
+			}
+			fw, err := zw.Create(item.name)
+			if err != nil {
+				serveError(w, http.StatusInternalServerError, fmt.Sprintf("Create zipped %s fail: %v", item.name, err))
+				return
+			}
+			err = p.WriteTo(fw, item.debug)
+			terror.Log(err)
+		}
+
+		// dump profile
+		fw, err := zw.Create("profile")
+		if err != nil {
+			serveError(w, http.StatusInternalServerError, fmt.Sprintf("Create zipped %s fail: %v", "profile", err))
+			return
+		}
+		if err := rpprof.StartCPUProfile(fw); err != nil {
+			serveError(w, http.StatusInternalServerError,
+				fmt.Sprintf("Could not enable CPU profiling: %s", err))
+			return
+		}
+		sec, err := strconv.ParseInt(r.FormValue("seconds"), 10, 64)
+		if sec <= 0 || err != nil {
+			sec = 10
+		}
+		sleep(w, time.Duration(sec)*time.Second)
+		rpprof.StopCPUProfile()
+
+		// dump config
+		fw, err = zw.Create("config")
+		if err != nil {
+			serveError(w, http.StatusInternalServerError, fmt.Sprintf("Create zipped %s fail: %v", "config", err))
+			return
+		}
+		js, err := json.MarshalIndent(config.GetGlobalConfig(), "", " ")
+		if err != nil {
+			serveError(w, http.StatusInternalServerError, fmt.Sprintf("get config info fail%v", err))
+			return
+		}
+		_, err = fw.Write(js)
+		terror.Log(err)
+
+		// dump version
+		fw, err = zw.Create("version")
+		if err != nil {
+			serveError(w, http.StatusInternalServerError, fmt.Sprintf("Create zipped %s fail: %v", "version", err))
+			return
+		}
+		_, err = fw.Write([]byte(printer.GetTiDBInfo()))
+		terror.Log(err)
+
+		err = zw.Close()
+		terror.Log(err)
+	})
+
 	log.Infof("Listening on %v for status and metrics report.", addr)
 	s.statusServer = &http.Server{Addr: addr, Handler: serverMux}
 	var err error