From 912cb5421de602c1794f9620524665502dc866b0 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 9 Oct 2018 13:23:30 +0800 Subject: [PATCH 1/3] domain: close slow query channel after closing session pool If slow query channel is closed before session pool, some session's goroutine may still writing to the channel. Writing to a closed channel would cause TiDB panic. --- domain/domain.go | 2 +- domain/topn_slow_query.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/domain/domain.go b/domain/domain.go index 865214c5972c3..cd6734b34c3e2 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -457,8 +457,8 @@ func (do *Domain) Close() { if do.etcdClient != nil { terror.Log(errors.Trace(do.etcdClient.Close())) } - do.slowQuery.Close() do.sysSessionPool.Close() + do.slowQuery.Close() do.wg.Wait() log.Info("[domain] close") } diff --git a/domain/topn_slow_query.go b/domain/topn_slow_query.go index 52251d34475f2..e86c61f192af9 100644 --- a/domain/topn_slow_query.go +++ b/domain/topn_slow_query.go @@ -196,6 +196,14 @@ func (q *topNSlowQueries) QueryTop(count int, kind ast.ShowSlowKind) []*SlowQuer } func (q *topNSlowQueries) Close() { + done := false + for !done { + select { + case <-q.ch: + default: + done = true + } + } close(q.ch) } From 477672a5f0adfef26408d4d4db9baa536222827e Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Wed, 10 Oct 2018 15:53:53 +0800 Subject: [PATCH 2/3] address comment --- domain/topn_slow_query.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/domain/topn_slow_query.go b/domain/topn_slow_query.go index e86c61f192af9..52251d34475f2 100644 --- a/domain/topn_slow_query.go +++ b/domain/topn_slow_query.go @@ -196,14 +196,6 @@ func (q *topNSlowQueries) QueryTop(count int, kind ast.ShowSlowKind) []*SlowQuer } func (q *topNSlowQueries) Close() { - done := false - for !done { - select { - case <-q.ch: - default: - done = true - } - } close(q.ch) } From 5ebd9282f2b42280e8b8e1fd7cc8751a3ea6e5a4 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 11 Oct 2018 12:56:45 +0800 Subject: [PATCH 3/3] address comment --- domain/domain.go | 6 ++++++ domain/topn_slow_query.go | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/domain/domain.go b/domain/domain.go index cd6734b34c3e2..8b0b98929aaad 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -335,6 +335,12 @@ func (do *Domain) Reload() error { // LogSlowQuery keeps topN recent slow queries in domain. func (do *Domain) LogSlowQuery(query *SlowQueryInfo) { + do.slowQuery.mu.RLock() + defer do.slowQuery.mu.RUnlock() + if do.slowQuery.mu.closed { + return + } + select { case do.slowQuery.ch <- query: default: diff --git a/domain/topn_slow_query.go b/domain/topn_slow_query.go index 52251d34475f2..f8ae69c0be67e 100644 --- a/domain/topn_slow_query.go +++ b/domain/topn_slow_query.go @@ -115,6 +115,11 @@ type topNSlowQueries struct { period time.Duration ch chan *SlowQueryInfo msgCh chan *showSlowMessage + + mu struct { + sync.RWMutex + closed bool + } } func newTopNSlowQueries(topN int, period time.Duration, queueSize int) *topNSlowQueries { @@ -196,6 +201,10 @@ func (q *topNSlowQueries) QueryTop(count int, kind ast.ShowSlowKind) []*SlowQuer } func (q *topNSlowQueries) Close() { + q.mu.Lock() + q.mu.closed = true + q.mu.Unlock() + close(q.ch) }