From a763b8a82306a691a3b513befbbd1b84a53df95e Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 5 Jun 2019 13:41:19 +0800 Subject: [PATCH 1/7] server: add http api to get some info of sub-optimal query --- server/http_status.go | 49 ++++--- server/sql_info_fetcher.go | 285 +++++++++++++++++++++++++++++++++++++ util/testkit/testkit.go | 25 +++- 3 files changed, 331 insertions(+), 28 deletions(-) create mode 100644 server/sql_info_fetcher.go diff --git a/server/http_status.go b/server/http_status.go index c7ca853b252de..a24e0ea1f12cc 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -34,7 +34,9 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/printer" "github.com/prometheus/client_golang/prometheus" @@ -49,6 +51,22 @@ func (s *Server) startStatusHTTP() { go s.startHTTPServer() } +func serveError(w http.ResponseWriter, status int, txt string) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Go-Pprof", "1") + w.Header().Del("Content-Disposition") + w.WriteHeader(status) + _, err := fmt.Fprintln(w, txt) + terror.Log(err) +} + +func sleepWithCtx(ctx context.Context, d time.Duration) { + select { + case <-time.After(d): + case <-ctx.Done(): + } +} + func (s *Server) startHTTPServer() { router := mux.NewRouter() @@ -123,26 +141,6 @@ func (s *Server) startHTTPServer() { serverMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) serverMux.HandleFunc("/debug/pprof/trace", pprof.Trace) - serveError := func(w http.ResponseWriter, status int, txt string) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Header().Set("X-Go-Pprof", "1") - w.Header().Del("Content-Disposition") - w.WriteHeader(status) - _, err := fmt.Fprintln(w, txt) - terror.Log(err) - } - - sleep := func(w http.ResponseWriter, d time.Duration) { - var clientGone <-chan bool - if cn, ok := w.(http.CloseNotifier); ok { - clientGone = cn.CloseNotify() - } - select { - case <-time.After(d): - case <-clientGone: - } - } - serverMux.HandleFunc("/debug/zip", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="tidb_debug"`+time.Now().Format("20060102150405")+".zip")) @@ -191,7 +189,7 @@ func (s *Server) startHTTPServer() { if sec <= 0 || err != nil { sec = 10 } - sleep(w, time.Duration(sec)*time.Second) + sleepWithCtx(r.Context(), time.Duration(sec)*time.Second) rpprof.StopCPUProfile() // dump config @@ -220,9 +218,16 @@ func (s *Server) startHTTPServer() { err = zw.Close() terror.Log(err) }) + se, err := session.CreateSession(tikvHandlerTool.Store) + if err == nil { + do := domain.GetDomain(se) + fetcher := sqlInfoFetcher{s: se, do: do} + serverMux.HandleFunc("/debug/sub-optimal-plan", fetcher.zipInfoForSQL) + } else { + logutil.Logger(context.Background()).Warn("create session for sql info fetch HTTP API", zap.Error(err)) + } var ( - err error httpRouterPage bytes.Buffer pathTemplate string ) diff --git a/server/sql_info_fetcher.go b/server/sql_info_fetcher.go new file mode 100644 index 0000000000000..af57ef0b362a8 --- /dev/null +++ b/server/sql_info_fetcher.go @@ -0,0 +1,285 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "archive/zip" + "context" + "encoding/json" + "fmt" + "net/http" + "runtime/pprof" + "strconv" + "strings" + "time" + + "github.com/pingcap/parser" + "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/model" + "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/statistics/handle" + "github.com/pingcap/tidb/util/sqlexec" + "github.com/pingcap/tidb/util/testkit" +) + +type sqlInfoFetcher struct { + do *domain.Domain + s session.Session +} + +type tableNamePair struct { + DBName string + TableName string +} + +type tableNameExtractor struct { + names map[tableNamePair]struct{} +} + +func (tne *tableNameExtractor) Enter(in ast.Node) (ast.Node, bool) { + if _, ok := in.(*ast.TableName); ok { + return in, true + } + return in, false +} + +func (tne *tableNameExtractor) Leave(in ast.Node) (ast.Node, bool) { + if t, ok := in.(*ast.TableName); ok { + tp := tableNamePair{DBName: t.Schema.L, TableName: t.Name.L} + if _, ok := tne.names[tp]; !ok { + tne.names[tp] = struct{}{} + } + } + return in, true +} + +func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) { + reqCtx := r.Context() + sql := r.FormValue("sql") + pprofTimeString := r.FormValue("pprof_time") + timeoutString := r.FormValue("timeout") + var ( + pprofTime int + timeout int + err error + ) + if pprofTimeString != "" { + pprofTime, err = strconv.Atoi(pprofTimeString) + } + if err != nil { + serveError(w, http.StatusBadRequest, "invalid value for pprof_time") + return + } + if pprofTime < 5 { + serveError(w, http.StatusBadRequest, "pprof time is too short") + } + if timeoutString != "" { + timeout, err = strconv.Atoi(timeoutString) + } + if err != nil { + serveError(w, http.StatusBadRequest, "invalid value for timeout") + return + } + if timeout < pprofTime { + timeout = pprofTime + } + pairs, err := sh.extractTableNames(sql) + if err != nil { + serveError(w, http.StatusBadRequest, fmt.Sprintf("invalid SQL text, err: %v", err)) + return + } + zw := zip.NewWriter(w) + defer func() { + terror.Log(zw.Close()) + }() + for pair := range pairs { + jsonTbl, err := sh.getStatsForTable(pair) + if err != nil { + terror.Log(err) + continue + } + statsFw, err := zw.Create(fmt.Sprintf("%v.%v.json", pair.DBName, pair.TableName)) + if err != nil { + terror.Log(err) + continue + } + data, err := json.Marshal(jsonTbl) + if err != nil { + terror.Log(err) + continue + } + _, err = statsFw.Write(data) + if err != nil { + terror.Log(err) + continue + } + } + for pair := range pairs { + err = sh.getShowCreateTable(pair, zw) + if err != nil { + serveError(w, http.StatusInternalServerError, fmt.Sprintf("get schema for %v.%v failed", pair.DBName, pair.TableName)) + return + } + } + // If we don't catch profile. We just get a explain result. + if pprofTime == 0 { + recordSets, err := sh.s.(sqlexec.SQLExecutor).Execute(reqCtx, fmt.Sprintf("explain %s", sql)) + if len(recordSets) > 0 { + defer terror.Call(recordSets[0].Close) + } + if err != nil { + serveError(w, http.StatusInternalServerError, fmt.Sprintf("execute SQL failed, err: %v", err)) + return + } + sRows, err := testkit.ResultSetToStringSlice(reqCtx, sh.s, recordSets[0]) + if err != nil { + terror.Log(err) + return + } + fw, err := zw.Create("explain.result") + if err != nil { + terror.Log(err) + return + } + for _, row := range sRows { + fmt.Fprintf(fw, "%s\n", strings.Join(row, "\t")) + } + } else { + // Otherwise we catch a profile and run `EXPLAIN ANALYZE` result. + ctx, cancelFunc := context.WithCancel(reqCtx) + timer := time.NewTimer(time.Second * time.Duration(timeout)) + resultChan := make(chan *explainAnalyzeResult) + go sh.getExplainAnalyze(ctx, sql, resultChan) + errChan := make(chan error) + go sh.catchCPUProfile(reqCtx, pprofTime, zw, errChan) + select { + case result := <-resultChan: + timer.Stop() + if result.err != nil { + serveError(w, http.StatusInternalServerError, fmt.Sprintf("explain analyze SQL failed, err: %v", err)) + return + } + if len(result.rows) == 0 { + break + } + fw, err := zw.Create("explain_analyze.result") + if err != nil { + terror.Log(err) + break + } + for _, row := range result.rows { + fmt.Fprintf(fw, "%s\n", strings.Join(row, "\t")) + } + case <-timer.C: + cancelFunc() + } + err = <-errChan + if err != nil { + serveError(w, http.StatusInternalServerError, fmt.Sprintf("catch cpu profile failed, error: %v", err)) + return + } + } +} + +type explainAnalyzeResult struct { + rows [][]string + err error +} + +func (sh *sqlInfoFetcher) getExplainAnalyze(ctx context.Context, sql string, resultChan chan<- *explainAnalyzeResult) { + recordSets, err := sh.s.(sqlexec.SQLExecutor).Execute(ctx, fmt.Sprintf("explain analyze %s", sql)) + if len(recordSets) > 0 { + defer terror.Call(recordSets[0].Close) + } + if err != nil { + resultChan <- &explainAnalyzeResult{err: err} + return + } + rows, err := testkit.ResultSetToStringSlice(ctx, sh.s, recordSets[0]) + if err != nil { + terror.Log(err) + rows = nil + return + } + resultChan <- &explainAnalyzeResult{rows: rows} +} + +func (sh *sqlInfoFetcher) catchCPUProfile(ctx context.Context, sec int, zw *zip.Writer, errChan chan<- error) { + // dump profile + fw, err := zw.Create("profile") + if err != nil { + errChan <- err + return + } + if err := pprof.StartCPUProfile(fw); err != nil { + errChan <- err + return + } + sleepWithCtx(ctx, time.Duration(sec)*time.Second) + pprof.StopCPUProfile() + errChan <- nil +} + +func (sh *sqlInfoFetcher) getStatsForTable(pair tableNamePair) (*handle.JSONTable, error) { + is := sh.do.InfoSchema() + h := sh.do.StatsHandle() + tbl, err := is.TableByName(model.NewCIStr(pair.DBName), model.NewCIStr(pair.TableName)) + if err != nil { + return nil, err + } + js, err := h.DumpStatsToJSON(pair.DBName, tbl.Meta(), nil) + return js, nil +} + +func (sh *sqlInfoFetcher) getShowCreateTable(pair tableNamePair, zw *zip.Writer) error { + recordSets, err := sh.s.(sqlexec.SQLExecutor).Execute(context.TODO(), fmt.Sprintf("show create table `%v`.`%v`", pair.DBName, pair.TableName)) + if len(recordSets) > 0 { + defer terror.Call(recordSets[0].Close) + } + if err != nil { + return err + } + sRows, err := testkit.ResultSetToStringSlice(context.Background(), sh.s, recordSets[0]) + if err != nil { + terror.Log(err) + return nil + } + fw, err := zw.Create(fmt.Sprintf("%v.%v.schema.txt", pair.DBName, pair.TableName)) + if err != nil { + terror.Log(err) + return nil + } + for _, row := range sRows { + fmt.Fprintf(fw, "%s\n", strings.Join(row, "\t")) + } + return nil +} + +func (sh *sqlInfoFetcher) extractTableNames(sql string) (map[tableNamePair]struct{}, error) { + p := parser.New() + charset, collation := sh.s.GetSessionVars().GetCharsetInfo() + stmts, _, err := p.Parse(sql, charset, collation) + if err != nil { + return nil, err + } + extractor := &tableNameExtractor{ + names: make(map[tableNamePair]struct{}), + } + for _, stmt := range stmts { + stmt.Accept(extractor) + } + return extractor.names, nil +} diff --git a/util/testkit/testkit.go b/util/testkit/testkit.go index c9e58d0cf9f58..ec443c6edd8fc 100644 --- a/util/testkit/testkit.go +++ b/util/testkit/testkit.go @@ -234,12 +234,16 @@ func (tk *TestKit) ResultSetToResult(rs sqlexec.RecordSet, comment check.Comment return tk.ResultSetToResultWithCtx(context.Background(), rs, comment) } -// ResultSetToResultWithCtx converts sqlexec.RecordSet to testkit.Result. -func (tk *TestKit) ResultSetToResultWithCtx(ctx context.Context, rs sqlexec.RecordSet, comment check.CommentInterface) *Result { - rows, err := session.GetRows4Test(ctx, tk.Se, rs) - tk.c.Assert(errors.ErrorStack(err), check.Equals, "", comment) +// ResultSetToStringSlice changes the RecordSet to [][]string. +func ResultSetToStringSlice(ctx context.Context, s session.Session, rs sqlexec.RecordSet) ([][]string, error) { + rows, err := session.GetRows4Test(ctx, s, rs) + if err != nil { + return nil, err + } err = rs.Close() - tk.c.Assert(errors.ErrorStack(err), check.Equals, "", comment) + if err != nil { + return nil, err + } sRows := make([][]string, len(rows)) for i := range rows { row := rows[i] @@ -250,11 +254,20 @@ func (tk *TestKit) ResultSetToResultWithCtx(ctx context.Context, rs sqlexec.Reco } else { d := row.GetDatum(j, &rs.Fields()[j].Column.FieldType) iRow[j], err = d.ToString() - tk.c.Assert(err, check.IsNil) + if err != nil { + return nil, err + } } } sRows[i] = iRow } + return sRows, nil +} + +// ResultSetToResultWithCtx converts sqlexec.RecordSet to testkit.Result. +func (tk *TestKit) ResultSetToResultWithCtx(ctx context.Context, rs sqlexec.RecordSet, comment check.CommentInterface) *Result { + sRows, err := ResultSetToStringSlice(ctx, tk.Se, rs) + tk.c.Check(err, check.IsNil, comment) return &Result{rows: sRows, c: tk.c, comment: comment} } From 0b32e299ac4f663ce69ec96f889cd407f07eca9a Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 5 Jun 2019 14:43:31 +0800 Subject: [PATCH 2/7] fix `make check` --- go.sum | 1 + server/sql_info_fetcher.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.sum b/go.sum index 53e26ae80c101..47d537b3ed45c 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,7 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= diff --git a/server/sql_info_fetcher.go b/server/sql_info_fetcher.go index af57ef0b362a8..c83ec41b5b569 100644 --- a/server/sql_info_fetcher.go +++ b/server/sql_info_fetcher.go @@ -168,6 +168,7 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) select { case result := <-resultChan: timer.Stop() + cancelFunc() if result.err != nil { serveError(w, http.StatusInternalServerError, fmt.Sprintf("explain analyze SQL failed, err: %v", err)) return @@ -241,7 +242,7 @@ func (sh *sqlInfoFetcher) getStatsForTable(pair tableNamePair) (*handle.JSONTabl return nil, err } js, err := h.DumpStatsToJSON(pair.DBName, tbl.Meta(), nil) - return js, nil + return js, err } func (sh *sqlInfoFetcher) getShowCreateTable(pair tableNamePair, zw *zip.Writer) error { From d2d94a5caafb7cedf31c89d2264a5a2ef0fd2ac5 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 5 Jun 2019 14:57:57 +0800 Subject: [PATCH 3/7] tiny change on file name --- server/sql_info_fetcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/sql_info_fetcher.go b/server/sql_info_fetcher.go index c83ec41b5b569..2dcb6adbed0af 100644 --- a/server/sql_info_fetcher.go +++ b/server/sql_info_fetcher.go @@ -149,7 +149,7 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) terror.Log(err) return } - fw, err := zw.Create("explain.result") + fw, err := zw.Create("explain.txt") if err != nil { terror.Log(err) return @@ -176,7 +176,7 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) if len(result.rows) == 0 { break } - fw, err := zw.Create("explain_analyze.result") + fw, err := zw.Create("explain_analyze.txt") if err != nil { terror.Log(err) break From 68e99eff4d9700d428e968146da5c8f9212195b6 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 5 Jun 2019 20:34:45 +0800 Subject: [PATCH 4/7] address comments --- server/http_status.go | 13 +++---------- server/sql_info_fetcher.go | 34 ++++++++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/server/http_status.go b/server/http_status.go index a24e0ea1f12cc..2dff937958b5e 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -34,9 +34,7 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" - "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/printer" "github.com/prometheus/client_golang/prometheus" @@ -218,18 +216,13 @@ func (s *Server) startHTTPServer() { err = zw.Close() terror.Log(err) }) - se, err := session.CreateSession(tikvHandlerTool.Store) - if err == nil { - do := domain.GetDomain(se) - fetcher := sqlInfoFetcher{s: se, do: do} - serverMux.HandleFunc("/debug/sub-optimal-plan", fetcher.zipInfoForSQL) - } else { - logutil.Logger(context.Background()).Warn("create session for sql info fetch HTTP API", zap.Error(err)) - } + fetcher := sqlInfoFetcher{store: tikvHandlerTool.Store} + serverMux.HandleFunc("/debug/sub-optimal-plan", fetcher.zipInfoForSQL) var ( httpRouterPage bytes.Buffer pathTemplate string + err error ) httpRouterPage.WriteString("TiDB Status and Metrics Report

TiDB Status and Metrics Report

") err = router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { diff --git a/server/sql_info_fetcher.go b/server/sql_info_fetcher.go index 2dcb6adbed0af..ad1d3ba1ee2b2 100644 --- a/server/sql_info_fetcher.go +++ b/server/sql_info_fetcher.go @@ -31,13 +31,15 @@ import ( "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/statistics/handle" + "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tidb/util/testkit" ) type sqlInfoFetcher struct { - do *domain.Domain - s session.Session + store tikv.Storage + do *domain.Domain + s session.Session } type tableNamePair struct { @@ -46,6 +48,7 @@ type tableNamePair struct { } type tableNameExtractor struct { + curDB string names map[tableNamePair]struct{} } @@ -59,6 +62,9 @@ func (tne *tableNameExtractor) Enter(in ast.Node) (ast.Node, bool) { func (tne *tableNameExtractor) Leave(in ast.Node) (ast.Node, bool) { if t, ok := in.(*ast.TableName); ok { tp := tableNamePair{DBName: t.Schema.L, TableName: t.Name.L} + if tp.DBName == "" { + tp.DBName = tne.curDB + } if _, ok := tne.names[tp]; !ok { tne.names[tp] = struct{}{} } @@ -67,14 +73,29 @@ func (tne *tableNameExtractor) Leave(in ast.Node) (ast.Node, bool) { } func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) { + var err error + sh.s, err = session.CreateSession(sh.store) + if err != nil { + serveError(w, http.StatusInternalServerError, fmt.Sprintf("create session failed, err: %v", err)) + return + } + defer sh.s.Close() + sh.do = domain.GetDomain(sh.s) reqCtx := r.Context() sql := r.FormValue("sql") pprofTimeString := r.FormValue("pprof_time") timeoutString := r.FormValue("timeout") + curDB := strings.ToLower(r.FormValue("current_db")) + if curDB != "" { + _, err = sh.s.Execute(reqCtx, fmt.Sprintf("use %v", curDB)) + if err != nil { + serveError(w, http.StatusInternalServerError, fmt.Sprintf("use database failed, err: %v", err)) + return + } + } var ( pprofTime int timeout int - err error ) if pprofTimeString != "" { pprofTime, err = strconv.Atoi(pprofTimeString) @@ -83,7 +104,7 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) serveError(w, http.StatusBadRequest, "invalid value for pprof_time") return } - if pprofTime < 5 { + if pprofTimeString != "" && pprofTime < 5 { serveError(w, http.StatusBadRequest, "pprof time is too short") } if timeoutString != "" { @@ -96,7 +117,7 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) if timeout < pprofTime { timeout = pprofTime } - pairs, err := sh.extractTableNames(sql) + pairs, err := sh.extractTableNames(sql, curDB) if err != nil { serveError(w, http.StatusBadRequest, fmt.Sprintf("invalid SQL text, err: %v", err)) return @@ -269,7 +290,7 @@ func (sh *sqlInfoFetcher) getShowCreateTable(pair tableNamePair, zw *zip.Writer) return nil } -func (sh *sqlInfoFetcher) extractTableNames(sql string) (map[tableNamePair]struct{}, error) { +func (sh *sqlInfoFetcher) extractTableNames(sql, curDB string) (map[tableNamePair]struct{}, error) { p := parser.New() charset, collation := sh.s.GetSessionVars().GetCharsetInfo() stmts, _, err := p.Parse(sql, charset, collation) @@ -277,6 +298,7 @@ func (sh *sqlInfoFetcher) extractTableNames(sql string) (map[tableNamePair]struc return nil, err } extractor := &tableNameExtractor{ + curDB: curDB, names: make(map[tableNamePair]struct{}), } for _, stmt := range stmts { From 11b99a2fa12381ae9d73897275bd07230d473086 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Mon, 10 Jun 2019 19:40:26 +0800 Subject: [PATCH 5/7] address comments --- server/sql_info_fetcher.go | 47 ++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/server/sql_info_fetcher.go b/server/sql_info_fetcher.go index ad1d3ba1ee2b2..81b5231d5b831 100644 --- a/server/sql_info_fetcher.go +++ b/server/sql_info_fetcher.go @@ -87,9 +87,9 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) timeoutString := r.FormValue("timeout") curDB := strings.ToLower(r.FormValue("current_db")) if curDB != "" { - _, err = sh.s.Execute(reqCtx, fmt.Sprintf("use %v", curDB)) + _, err = sh.s.Execute(reqCtx, "use %v" + curDB) if err != nil { - serveError(w, http.StatusInternalServerError, fmt.Sprintf("use database failed, err: %v", err)) + serveError(w, http.StatusInternalServerError, fmt.Sprintf("use database %v failed, err: %v", curDB, err)) return } } @@ -99,20 +99,20 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) ) if pprofTimeString != "" { pprofTime, err = strconv.Atoi(pprofTimeString) - } - if err != nil { - serveError(w, http.StatusBadRequest, "invalid value for pprof_time") - return + if err != nil { + serveError(w, http.StatusBadRequest, "invalid value for pprof_time, please input a int value larger than 5") + return + } } if pprofTimeString != "" && pprofTime < 5 { - serveError(w, http.StatusBadRequest, "pprof time is too short") + serveError(w, http.StatusBadRequest, "pprof time is too short, please input a int value larger than 5") } if timeoutString != "" { timeout, err = strconv.Atoi(timeoutString) - } - if err != nil { - serveError(w, http.StatusBadRequest, "invalid value for timeout") - return + if err != nil { + serveError(w, http.StatusBadRequest, "invalid value for timeout") + return + } } if timeout < pprofTime { timeout = pprofTime @@ -129,6 +129,7 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) for pair := range pairs { jsonTbl, err := sh.getStatsForTable(pair) if err != nil { + err = sh.writeErrFile(zw, fmt.Sprintf("%v.%v.stats.err.txt", pair.DBName, pair.TableName), err) terror.Log(err) continue } @@ -139,11 +140,13 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) } data, err := json.Marshal(jsonTbl) if err != nil { + err = sh.writeErrFile(zw, fmt.Sprintf("%v.%v.stats.err.txt", pair.DBName, pair.TableName), err) terror.Log(err) continue } _, err = statsFw.Write(data) if err != nil { + err = sh.writeErrFile(zw, fmt.Sprintf("%v.%v.stats.err.txt", pair.DBName, pair.TableName), err) terror.Log(err) continue } @@ -151,7 +154,8 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) for pair := range pairs { err = sh.getShowCreateTable(pair, zw) if err != nil { - serveError(w, http.StatusInternalServerError, fmt.Sprintf("get schema for %v.%v failed", pair.DBName, pair.TableName)) + err = sh.writeErrFile(zw, fmt.Sprintf("%v.%v.schema.err.txt", pair.DBName, pair.TableName), err) + terror.Log(err) return } } @@ -162,11 +166,13 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) defer terror.Call(recordSets[0].Close) } if err != nil { - serveError(w, http.StatusInternalServerError, fmt.Sprintf("execute SQL failed, err: %v", err)) + err = sh.writeErrFile(zw, "explain.err.txt", err) + terror.Log(err) return } sRows, err := testkit.ResultSetToStringSlice(reqCtx, sh.s, recordSets[0]) if err != nil { + err = sh.writeErrFile(zw, "explain.err.txt", err) terror.Log(err) return } @@ -191,7 +197,8 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) timer.Stop() cancelFunc() if result.err != nil { - serveError(w, http.StatusInternalServerError, fmt.Sprintf("explain analyze SQL failed, err: %v", err)) + err = sh.writeErrFile(zw, "explain_analyze.err.txt", result.err) + terror.Log(err) return } if len(result.rows) == 0 { @@ -210,12 +217,22 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) } err = <-errChan if err != nil { - serveError(w, http.StatusInternalServerError, fmt.Sprintf("catch cpu profile failed, error: %v", err)) + err = sh.writeErrFile(zw, "profile.err.txt", err) + terror.Log(err) return } } } +func (sh *sqlInfoFetcher) writeErrFile(zw *zip.Writer, name string, err error) error { + fw, err1 := zw.Create(name) + if err1 != nil { + return err1 + } + fmt.Fprintf(fw, "error: %v", err) + return nil +} + type explainAnalyzeResult struct { rows [][]string err error From 6238febad2c45c1e992734299697f96e1fc1b65d Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Mon, 24 Jun 2019 11:28:35 +0800 Subject: [PATCH 6/7] address comments --- server/sql_info_fetcher.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/sql_info_fetcher.go b/server/sql_info_fetcher.go index 81b5231d5b831..8f07a340c13d6 100644 --- a/server/sql_info_fetcher.go +++ b/server/sql_info_fetcher.go @@ -24,6 +24,7 @@ import ( "strings" "time" + "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" @@ -87,7 +88,7 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) timeoutString := r.FormValue("timeout") curDB := strings.ToLower(r.FormValue("current_db")) if curDB != "" { - _, err = sh.s.Execute(reqCtx, "use %v" + curDB) + _, err = sh.s.Execute(reqCtx, "use %v"+curDB) if err != nil { serveError(w, http.StatusInternalServerError, fmt.Sprintf("use database %v failed, err: %v", curDB, err)) return @@ -314,12 +315,13 @@ func (sh *sqlInfoFetcher) extractTableNames(sql, curDB string) (map[tableNamePai if err != nil { return nil, err } + if len(stmts) > 1 { + return nil, errors.Errorf("Only 1 statement is allowed") + } extractor := &tableNameExtractor{ curDB: curDB, names: make(map[tableNamePair]struct{}), } - for _, stmt := range stmts { - stmt.Accept(extractor) - } + stmts[0].Accept(extractor) return extractor.names, nil } From c152e7e4a3d964d3e76f226135b1de7ed4e20927 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Mon, 24 Jun 2019 11:34:38 +0800 Subject: [PATCH 7/7] fix build --- server/http_status.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/http_status.go b/server/http_status.go index 8a039050c0793..b2114a4ae13a2 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -16,6 +16,7 @@ package server import ( "archive/zip" "bytes" + "context" "encoding/json" "fmt" "net"