Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

config, util.kvcache: support the memory guard to prevent OOM for the plan cache #8339

Merged
merged 5 commits into from
Nov 22, 2018
Merged

config, util.kvcache: support the memory guard to prevent OOM for the plan cache #8339

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
33bb20a
config, util.kvcache: support the memory guard to prevent OOM for the…
dbjoa Nov 22, 2018
acd7ce5
add the bench test and the license info
dbjoa Nov 22, 2018
256367a
1. adjuest the configuration values if they are in out of ranges
dbjoa Nov 22, 2018
20b6f62
fix the CI errors
dbjoa Nov 22, 2018
1c20285
Merge branch 'master' into support-byte-sized-capacity-for-the-plan-c…
zz-jason Nov 22, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ type Status struct {
// Performance is the performance section of the config.
type Performance struct {
MaxProcs uint `toml:"max-procs" json:"max-procs"`
MaxMemory uint64 `toml:"max-memory" json:"max-memory"`
TCPKeepAlive bool `toml:"tcp-keep-alive" json:"tcp-keep-alive"`
CrossJoin bool `toml:"cross-join" json:"cross-join"`
StatsLease string `toml:"stats-lease" json:"stats-lease"`
Expand Down Expand Up @@ -184,8 +185,9 @@ type TxnLocalLatches struct {

// PreparedPlanCache is the PreparedPlanCache section of the config.
type PreparedPlanCache struct {
Enabled bool `toml:"enabled" json:"enabled"`
Capacity uint `toml:"capacity" json:"capacity"`
Enabled bool `toml:"enabled" json:"enabled"`
Capacity uint `toml:"capacity" json:"capacity"`
MemoryGuardRatio float64 `toml:"memory-guard-ratio" json:"memory-guard-ratio"`
}

// OpenTracing is the opentracing section of the config.
Expand Down Expand Up @@ -287,6 +289,7 @@ var defaultConf = Config{
MetricsInterval: 15,
},
Performance: Performance{
MaxMemory: 0,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the configed MaxMemory is larger than the total memory of the machine, should we adjust this value to the total memory of that machine?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The updated PR should address the issue.

TCPKeepAlive: true,
CrossJoin: true,
StatsLease: "3s",
Expand All @@ -306,8 +309,9 @@ var defaultConf = Config{
HeaderTimeout: 5,
},
PreparedPlanCache: PreparedPlanCache{
Enabled: false,
Capacity: 100,
Enabled: false,
Capacity: 100,
MemoryGuardRatio: 0.1,
},
OpenTracing: OpenTracing{
Enable: false,
Expand Down
3 changes: 3 additions & 0 deletions config/config.toml.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,8 @@ metrics-interval = 15
[performance]
# Max CPUs to use, 0 use number of CPUs in the machine.
max-procs = 0
# Max memory size to use, 0 use the total usable memory in the machine.
max-memory = 0
# StmtCountLimit limits the max count of statement inside a transaction.
stmt-count-limit = 5000

Expand Down Expand Up @@ -162,6 +164,7 @@ header-timeout = 5
[prepared-plan-cache]
enabled = false
capacity = 100
memory-guard-ratio = 0.1

[opentracing]
# Enable opentracing.
Expand Down
77 changes: 75 additions & 2 deletions executor/prepared_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/pingcap/tidb/executor"
"github.com/pingcap/tidb/metrics"
plannercore "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/testkit"
dto "github.com/prometheus/client_model/go"
"golang.org/x/net/context"
Expand All @@ -31,15 +32,23 @@ import (
func (s *testSuite) TestPrepared(c *C) {
orgEnable := plannercore.PreparedPlanCacheEnabled()
orgCapacity := plannercore.PreparedPlanCacheCapacity
orgMemGuardRatio := plannercore.PreparedPlanCacheMemoryGuardRatio
orgMaxMemory := plannercore.PreparedPlanCacheMaxMemory
defer func() {
plannercore.SetPreparedPlanCache(orgEnable)
plannercore.PreparedPlanCacheCapacity = orgCapacity
plannercore.PreparedPlanCacheMemoryGuardRatio = orgMemGuardRatio
plannercore.PreparedPlanCacheMaxMemory = orgMaxMemory
}()
flags := []bool{false, true}
ctx := context.Background()
for _, flag := range flags {
var err error
plannercore.SetPreparedPlanCache(flag)
plannercore.PreparedPlanCacheCapacity = 100
plannercore.PreparedPlanCacheMemoryGuardRatio = 0.1
plannercore.PreparedPlanCacheMaxMemory, err = memory.MemTotal()
c.Assert(err, IsNil)
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists prepare_test")
Expand All @@ -49,7 +58,7 @@ func (s *testSuite) TestPrepared(c *C) {
tk.MustExec(`prepare stmt_test_1 from 'select id from prepare_test where id > ?'; set @a = 1; execute stmt_test_1 using @a;`)
tk.MustExec(`prepare stmt_test_2 from 'select 1'`)
// Prepare multiple statement is not allowed.
_, err := tk.Exec(`prepare stmt_test_3 from 'select id from prepare_test where id > ?;select id from prepare_test where id > ?;'`)
_, err = tk.Exec(`prepare stmt_test_3 from 'select id from prepare_test where id > ?;select id from prepare_test where id > ?;'`)
c.Assert(executor.ErrPrepareMulti.Equal(err), IsTrue)
// The variable count does not match.
_, err = tk.Exec(`prepare stmt_test_4 from 'select id from prepare_test where id > ? and id < ?'; set @a = 1; execute stmt_test_4 using @a;`)
Expand Down Expand Up @@ -215,15 +224,23 @@ func (s *testSuite) TestPrepared(c *C) {
func (s *testSuite) TestPreparedLimitOffset(c *C) {
orgEnable := plannercore.PreparedPlanCacheEnabled()
orgCapacity := plannercore.PreparedPlanCacheCapacity
orgMemGuardRatio := plannercore.PreparedPlanCacheMemoryGuardRatio
orgMaxMemory := plannercore.PreparedPlanCacheMaxMemory
defer func() {
plannercore.SetPreparedPlanCache(orgEnable)
plannercore.PreparedPlanCacheCapacity = orgCapacity
plannercore.PreparedPlanCacheMemoryGuardRatio = orgMemGuardRatio
plannercore.PreparedPlanCacheMaxMemory = orgMaxMemory
}()
flags := []bool{false, true}
ctx := context.Background()
for _, flag := range flags {
var err error
plannercore.SetPreparedPlanCache(flag)
plannercore.PreparedPlanCacheCapacity = 100
plannercore.PreparedPlanCacheMemoryGuardRatio = 0.1
plannercore.PreparedPlanCacheMaxMemory, err = memory.MemTotal()
c.Assert(err, IsNil)
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists prepare_test")
Expand All @@ -238,7 +255,7 @@ func (s *testSuite) TestPreparedLimitOffset(c *C) {
r.Check(testkit.Rows("2"))

tk.MustExec(`set @c="-1"`)
_, err := tk.Exec("execute stmt_test_1 using @c, @c")
_, err = tk.Exec("execute stmt_test_1 using @c, @c")
c.Assert(plannercore.ErrWrongArguments.Equal(err), IsTrue)

stmtID, _, _, err := tk.Se.PrepareStmt("select id from prepare_test limit ?")
Expand All @@ -251,14 +268,22 @@ func (s *testSuite) TestPreparedLimitOffset(c *C) {
func (s *testSuite) TestPreparedNullParam(c *C) {
orgEnable := plannercore.PreparedPlanCacheEnabled()
orgCapacity := plannercore.PreparedPlanCacheCapacity
orgMemGuardRatio := plannercore.PreparedPlanCacheMemoryGuardRatio
orgMaxMemory := plannercore.PreparedPlanCacheMaxMemory
defer func() {
plannercore.SetPreparedPlanCache(orgEnable)
plannercore.PreparedPlanCacheCapacity = orgCapacity
plannercore.PreparedPlanCacheMemoryGuardRatio = orgMemGuardRatio
plannercore.PreparedPlanCacheMaxMemory = orgMaxMemory
}()
flags := []bool{false, true}
for _, flag := range flags {
var err error
plannercore.SetPreparedPlanCache(flag)
plannercore.PreparedPlanCacheCapacity = 100
plannercore.PreparedPlanCacheMemoryGuardRatio = 0.1
plannercore.PreparedPlanCacheMaxMemory, err = memory.MemTotal()
c.Assert(err, IsNil)
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
Expand Down Expand Up @@ -315,14 +340,22 @@ func (s *testSuite) TestPrepareMaxParamCountCheck(c *C) {
func (s *testSuite) TestPrepareWithAggregation(c *C) {
orgEnable := plannercore.PreparedPlanCacheEnabled()
orgCapacity := plannercore.PreparedPlanCacheCapacity
orgMemGuardRatio := plannercore.PreparedPlanCacheMemoryGuardRatio
orgMaxMemory := plannercore.PreparedPlanCacheMaxMemory
defer func() {
plannercore.SetPreparedPlanCache(orgEnable)
plannercore.PreparedPlanCacheCapacity = orgCapacity
plannercore.PreparedPlanCacheMemoryGuardRatio = orgMemGuardRatio
plannercore.PreparedPlanCacheMaxMemory = orgMaxMemory
}()
flags := []bool{false, true}
for _, flag := range flags {
var err error
plannercore.SetPreparedPlanCache(flag)
plannercore.PreparedPlanCacheCapacity = 100
plannercore.PreparedPlanCacheMemoryGuardRatio = 0.1
plannercore.PreparedPlanCacheMaxMemory, err = memory.MemTotal()
c.Assert(err, IsNil)
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
Expand Down Expand Up @@ -352,14 +385,22 @@ func generateBatchSQL(paramCount int) (sql string, paramSlice []interface{}) {
func (s *testSuite) TestPreparedIssue7579(c *C) {
orgEnable := plannercore.PreparedPlanCacheEnabled()
orgCapacity := plannercore.PreparedPlanCacheCapacity
orgMemGuardRatio := plannercore.PreparedPlanCacheMemoryGuardRatio
orgMaxMemory := plannercore.PreparedPlanCacheMaxMemory
defer func() {
plannercore.SetPreparedPlanCache(orgEnable)
plannercore.PreparedPlanCacheCapacity = orgCapacity
plannercore.PreparedPlanCacheMemoryGuardRatio = orgMemGuardRatio
plannercore.PreparedPlanCacheMaxMemory = orgMaxMemory
}()
flags := []bool{false, true}
for _, flag := range flags {
var err error
plannercore.SetPreparedPlanCache(flag)
plannercore.PreparedPlanCacheCapacity = 100
plannercore.PreparedPlanCacheMemoryGuardRatio = 0.1
plannercore.PreparedPlanCacheMaxMemory, err = memory.MemTotal()
c.Assert(err, IsNil)
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
Expand Down Expand Up @@ -395,17 +436,25 @@ func (s *testSuite) TestPreparedIssue7579(c *C) {
func (s *testSuite) TestPreparedInsert(c *C) {
orgEnable := plannercore.PreparedPlanCacheEnabled()
orgCapacity := plannercore.PreparedPlanCacheCapacity
orgMemGuardRatio := plannercore.PreparedPlanCacheMemoryGuardRatio
orgMaxMemory := plannercore.PreparedPlanCacheMaxMemory
defer func() {
plannercore.SetPreparedPlanCache(orgEnable)
plannercore.PreparedPlanCacheCapacity = orgCapacity
plannercore.PreparedPlanCacheMemoryGuardRatio = orgMemGuardRatio
plannercore.PreparedPlanCacheMaxMemory = orgMaxMemory
}()
metrics.PlanCacheCounter.Reset()
counter := metrics.PlanCacheCounter.WithLabelValues("prepare")
pb := &dto.Metric{}
flags := []bool{false, true}
for _, flag := range flags {
var err error
plannercore.SetPreparedPlanCache(flag)
plannercore.PreparedPlanCacheCapacity = 100
plannercore.PreparedPlanCacheMemoryGuardRatio = 0.1
plannercore.PreparedPlanCacheMaxMemory, err = memory.MemTotal()
c.Assert(err, IsNil)
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists prepare_test")
Expand Down Expand Up @@ -469,17 +518,25 @@ func (s *testSuite) TestPreparedInsert(c *C) {
func (s *testSuite) TestPreparedUpdate(c *C) {
orgEnable := plannercore.PreparedPlanCacheEnabled()
orgCapacity := plannercore.PreparedPlanCacheCapacity
orgMemGuardRatio := plannercore.PreparedPlanCacheMemoryGuardRatio
orgMaxMemory := plannercore.PreparedPlanCacheMaxMemory
defer func() {
plannercore.SetPreparedPlanCache(orgEnable)
plannercore.PreparedPlanCacheCapacity = orgCapacity
plannercore.PreparedPlanCacheMemoryGuardRatio = orgMemGuardRatio
plannercore.PreparedPlanCacheMaxMemory = orgMaxMemory
}()
metrics.PlanCacheCounter.Reset()
counter := metrics.PlanCacheCounter.WithLabelValues("prepare")
pb := &dto.Metric{}
flags := []bool{false, true}
for _, flag := range flags {
var err error
plannercore.SetPreparedPlanCache(flag)
plannercore.PreparedPlanCacheCapacity = 100
plannercore.PreparedPlanCacheMemoryGuardRatio = 0.1
plannercore.PreparedPlanCacheMaxMemory, err = memory.MemTotal()
c.Assert(err, IsNil)
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists prepare_test")
Expand Down Expand Up @@ -520,17 +577,25 @@ func (s *testSuite) TestPreparedUpdate(c *C) {
func (s *testSuite) TestPreparedDelete(c *C) {
orgEnable := plannercore.PreparedPlanCacheEnabled()
orgCapacity := plannercore.PreparedPlanCacheCapacity
orgMemGuardRatio := plannercore.PreparedPlanCacheMemoryGuardRatio
orgMaxMemory := plannercore.PreparedPlanCacheMaxMemory
defer func() {
plannercore.SetPreparedPlanCache(orgEnable)
plannercore.PreparedPlanCacheCapacity = orgCapacity
plannercore.PreparedPlanCacheMemoryGuardRatio = orgMemGuardRatio
plannercore.PreparedPlanCacheMaxMemory = orgMaxMemory
}()
metrics.PlanCacheCounter.Reset()
counter := metrics.PlanCacheCounter.WithLabelValues("prepare")
pb := &dto.Metric{}
flags := []bool{false, true}
for _, flag := range flags {
var err error
plannercore.SetPreparedPlanCache(flag)
plannercore.PreparedPlanCacheCapacity = 100
plannercore.PreparedPlanCacheMemoryGuardRatio = 0.1
plannercore.PreparedPlanCacheMaxMemory, err = memory.MemTotal()
c.Assert(err, IsNil)
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists prepare_test")
Expand Down Expand Up @@ -571,12 +636,20 @@ func (s *testSuite) TestPreparedDelete(c *C) {
func (s *testSuite) TestPrepareDealloc(c *C) {
orgEnable := plannercore.PreparedPlanCacheEnabled()
orgCapacity := plannercore.PreparedPlanCacheCapacity
orgMemGuardRatio := plannercore.PreparedPlanCacheMemoryGuardRatio
orgMaxMemory := plannercore.PreparedPlanCacheMaxMemory
defer func() {
plannercore.SetPreparedPlanCache(orgEnable)
plannercore.PreparedPlanCacheCapacity = orgCapacity
plannercore.PreparedPlanCacheMemoryGuardRatio = orgMemGuardRatio
plannercore.PreparedPlanCacheMaxMemory = orgMaxMemory
}()
var err error
plannercore.SetPreparedPlanCache(true)
plannercore.PreparedPlanCacheCapacity = 3
plannercore.PreparedPlanCacheMemoryGuardRatio = 0.1
plannercore.PreparedPlanCacheMaxMemory, err = memory.MemTotal()
c.Assert(err, IsNil)

tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ require (
github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 // indirect
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d // indirect
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 // indirect
github.com/shirou/gopsutil v2.18.10+incompatible
github.com/sirupsen/logrus v1.2.0
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72
github.com/twinj/uuid v1.0.0
Expand Down
3 changes: 3 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ github.com/pingcap/tidb-tools v0.0.0-20181101090416-cfac1096162e h1:LKGiK9RwOntq
github.com/pingcap/tidb-tools v0.0.0-20181101090416-cfac1096162e/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tidb-tools v0.0.0-20181112132202-4860a0d5de03 h1:xVuo5U+l6XAWHsb+xhkZ8zz3jerIwDfCHAO6kR2Kaog=
github.com/pingcap/tidb-tools v0.0.0-20181112132202-4860a0d5de03/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tidb-tools v2.1.0-rc.5+incompatible h1:x+vS+/RXiJsX2lvED0zGSP08Yc2e6r2WtQCCmwc9ASg=
github.com/pingcap/tipb v0.0.0-20171213095807-07ff5b094233/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323 h1:mRKKzRjDNaUNPnAkPAHnRqpNmwNWBX1iA+hxlmvQ93I=
github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
Expand Down Expand Up @@ -257,6 +258,8 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/shirou/gopsutil v2.18.10+incompatible h1:cy84jW6EVRPa5g9HAHrlbxMSIjBhDSX0OFYyMYminYs=
github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/siddontang/go v0.0.0-20170517070808-cb568a3e5cc0/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
github.com/sirupsen/logrus v0.0.0-20170323161349-3bcb09397d6d/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
Expand Down
4 changes: 4 additions & 0 deletions planner/core/cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ var (
preparedPlanCacheEnabledValue int32
// PreparedPlanCacheCapacity stores the global config "prepared-plan-cache-capacity".
PreparedPlanCacheCapacity uint
// PreparedPlanCacheMemoryGuardRatio stores the global config "prepared-plan-cache-memory-guard-ratio".
PreparedPlanCacheMemoryGuardRatio float64
// PreparedPlanCacheMaxMemory stores the max memory size defined in the global config "performance-max-memory".
PreparedPlanCacheMaxMemory uint64
)

const (
Expand Down
8 changes: 8 additions & 0 deletions planner/core/point_get_plan_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
. "github.com/pingcap/check"
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/testkit"
"github.com/pingcap/tidb/util/testleak"
dto "github.com/prometheus/client_model/go"
Expand All @@ -34,14 +35,21 @@ func (s *testPointGetSuite) TestPointGetPlanCache(c *C) {
tk := testkit.NewTestKit(c, store)
orgEnable := core.PreparedPlanCacheEnabled()
orgCapacity := core.PreparedPlanCacheCapacity
orgMemGuardRatio := core.PreparedPlanCacheMemoryGuardRatio
orgMaxMemory := core.PreparedPlanCacheMaxMemory
defer func() {
dom.Close()
store.Close()
core.SetPreparedPlanCache(orgEnable)
core.PreparedPlanCacheCapacity = orgCapacity
core.PreparedPlanCacheMemoryGuardRatio = orgMemGuardRatio
core.PreparedPlanCacheMaxMemory = orgMaxMemory
}()
core.SetPreparedPlanCache(true)
core.PreparedPlanCacheCapacity = 100
core.PreparedPlanCacheMemoryGuardRatio = 0.1
core.PreparedPlanCacheMaxMemory, err = memory.MemTotal()
c.Assert(err, IsNil)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int primary key, b int, c int, key idx_bc(b,c))")
Expand Down
Loading