Merge branch 'master' into get_tz_name_from_zoneinfo

distsql/distsql.go

@@ -16,6 +16,7 @@ package distsql
 import (
 	"github.com/juju/errors"
 	"github.com/pingcap/tidb/kv"
+	"github.com/pingcap/tidb/metrics"
 	"github.com/pingcap/tidb/sessionctx"
 	"github.com/pingcap/tidb/statistics"
 	"github.com/pingcap/tidb/types"
@@ -54,6 +55,10 @@ func Select(ctx context.Context, sctx sessionctx.Context, kvReq *kv.Request, fie
 		}, nil
 	}
 
+	label := metrics.LblGeneral
+	if sctx.GetSessionVars().InRestrictedSQL {
+		label = metrics.LblInternal
+	}
 	return &selectResult{
 		label:      "dag",
 		resp:       resp,
@@ -63,21 +68,28 @@ func Select(ctx context.Context, sctx sessionctx.Context, kvReq *kv.Request, fie
 		fieldTypes: fieldTypes,
 		ctx:        sctx,
 		feedback:   fb,
+		sqlType:    label,
 	}, nil
 }
 
 // Analyze do a analyze request.
-func Analyze(ctx context.Context, client kv.Client, kvReq *kv.Request, vars *kv.Variables) (SelectResult, error) {
+func Analyze(ctx context.Context, client kv.Client, kvReq *kv.Request, vars *kv.Variables,
+	isRestrict bool) (SelectResult, error) {
 	resp := client.Send(ctx, kvReq, vars)
 	if resp == nil {
 		return nil, errors.New("client returns nil response")
 	}
+	label := metrics.LblGeneral
+	if isRestrict {
+		label = metrics.LblInternal
+	}
 	result := &selectResult{
 		label:    "analyze",
 		resp:     resp,
 		results:  make(chan resultWithErr, kvReq.Concurrency),
 		closed:   make(chan struct{}),
 		feedback: statistics.NewQueryFeedback(0, nil, 0, false),
+		sqlType:  label,
 	}
 	return result, nil
 }
@@ -94,6 +106,7 @@ func Checksum(ctx context.Context, client kv.Client, kvReq *kv.Request, vars *kv
 		results:  make(chan resultWithErr, kvReq.Concurrency),
 		closed:   make(chan struct{}),
 		feedback: statistics.NewQueryFeedback(0, nil, 0, false),
+		sqlType:  metrics.LblGeneral,
 	}
 	return result, nil
 }

distsql/distsql_test.go

@@ -64,6 +64,7 @@ func (s *testSuite) TestSelectNormal(c *C) {
 	result, ok := response.(*selectResult)
 	c.Assert(ok, IsTrue)
 	c.Assert(result.label, Equals, "dag")
+	c.Assert(result.sqlType, Equals, "general")
 	c.Assert(result.rowLen, Equals, len(colTypes))
 
 	response.Fetch(context.TODO())
@@ -143,12 +144,13 @@ func (s *testSuite) TestAnalyze(c *C) {
 		Build()
 	c.Assert(err, IsNil)
 
-	response, err := Analyze(context.TODO(), s.sctx.GetClient(), request, kv.DefaultVars)
+	response, err := Analyze(context.TODO(), s.sctx.GetClient(), request, kv.DefaultVars, true)
 	c.Assert(err, IsNil)
 
 	result, ok := response.(*selectResult)
 	c.Assert(ok, IsTrue)
 	c.Assert(result.label, Equals, "analyze")
+	c.Assert(result.sqlType, Equals, "internal")
 
 	response.Fetch(context.TODO())
 

distsql/select_result.go

@@ -67,6 +67,7 @@ type selectResult struct {
 
 	feedback     *statistics.QueryFeedback
 	partialCount int64 // number of partial results.
+	sqlType      string
 }
 
 func (r *selectResult) Fetch(ctx context.Context) {
@@ -78,7 +79,7 @@ func (r *selectResult) fetch(ctx context.Context) {
 	defer func() {
 		close(r.results)
 		duration := time.Since(startTime)
-		metrics.DistSQLQueryHistgram.WithLabelValues(r.label).Observe(duration.Seconds())
+		metrics.DistSQLQueryHistgram.WithLabelValues(r.label, r.sqlType).Observe(duration.Seconds())
 	}()
 	for {
 		resultSubset, err := r.resp.Next(ctx)

executor/adapter.go

@@ -354,15 +354,19 @@ func (a *ExecStmt) logSlowQuery(txnTS uint64, succ bool) {
 	if len(sessVars.StmtCtx.IndexIDs) > 0 {
 		indexIDs = strings.Replace(fmt.Sprintf("index_ids:%v ", a.Ctx.GetSessionVars().StmtCtx.IndexIDs), " ", ",", -1)
 	}
-	user := a.Ctx.GetSessionVars().User
+	user := sessVars.User
+	var internal string
+	if sessVars.InRestrictedSQL {
+		internal = "[INTERNAL] "
+	}
 	if costTime < threshold {
 		logutil.SlowQueryLogger.Debugf(
-			"[QUERY] cost_time:%v %s succ:%v con:%v user:%s txn_start_ts:%v database:%v %v%vsql:%v",
-			costTime, sessVars.StmtCtx.GetExecDetails(), succ, connID, user, txnTS, currentDB, tableIDs, indexIDs, sql)
+			"[QUERY] %vcost_time:%v %s succ:%v con:%v user:%s txn_start_ts:%v database:%v %v%vsql:%v",
+			internal, costTime, sessVars.StmtCtx.GetExecDetails(), succ, connID, user, txnTS, currentDB, tableIDs, indexIDs, sql)
 	} else {
 		logutil.SlowQueryLogger.Warnf(
-			"[SLOW_QUERY] cost_time:%v %s succ:%v con:%v user:%s txn_start_ts:%v database:%v %v%vsql:%v",
-			costTime, sessVars.StmtCtx.GetExecDetails(), succ, connID, user, txnTS, currentDB, tableIDs, indexIDs, sql)
+			"[SLOW_QUERY] %vcost_time:%v %s succ:%v con:%v user:%s txn_start_ts:%v database:%v %v%vsql:%v",
+			internal, costTime, sessVars.StmtCtx.GetExecDetails(), succ, connID, user, txnTS, currentDB, tableIDs, indexIDs, sql)
 	}
 }
 

executor/analyze.go

@@ -176,7 +176,7 @@ func (e *AnalyzeIndexExec) open() error {
 		SetConcurrency(e.concurrency).
 		Build()
 	ctx := context.TODO()
-	e.result, err = distsql.Analyze(ctx, e.ctx.GetClient(), kvReq, e.ctx.GetSessionVars().KVVars)
+	e.result, err = distsql.Analyze(ctx, e.ctx.GetClient(), kvReq, e.ctx.GetSessionVars().KVVars, e.ctx.GetSessionVars().InRestrictedSQL)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -295,7 +295,7 @@ func (e *AnalyzeColumnsExec) buildResp(ranges []*ranger.Range) (distsql.SelectRe
 		return nil, errors.Trace(err)
 	}
 	ctx := context.TODO()
-	result, err := distsql.Analyze(ctx, e.ctx.GetClient(), kvReq, e.ctx.GetSessionVars().KVVars)
+	result, err := distsql.Analyze(ctx, e.ctx.GetClient(), kvReq, e.ctx.GetSessionVars().KVVars, e.ctx.GetSessionVars().InRestrictedSQL)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}

expression/builtin.go

@@ -572,14 +572,15 @@ var funcs = map[string]functionClass{
 	ast.ValidatePasswordStrength: &validatePasswordStrengthFunctionClass{baseFunctionClass{ast.ValidatePasswordStrength, 1, 1}},
 
 	// json functions
-	ast.JSONType:    &jsonTypeFunctionClass{baseFunctionClass{ast.JSONType, 1, 1}},
-	ast.JSONExtract: &jsonExtractFunctionClass{baseFunctionClass{ast.JSONExtract, 2, -1}},
-	ast.JSONUnquote: &jsonUnquoteFunctionClass{baseFunctionClass{ast.JSONUnquote, 1, 1}},
-	ast.JSONSet:     &jsonSetFunctionClass{baseFunctionClass{ast.JSONSet, 3, -1}},
-	ast.JSONInsert:  &jsonInsertFunctionClass{baseFunctionClass{ast.JSONInsert, 3, -1}},
-	ast.JSONReplace: &jsonReplaceFunctionClass{baseFunctionClass{ast.JSONReplace, 3, -1}},
-	ast.JSONRemove:  &jsonRemoveFunctionClass{baseFunctionClass{ast.JSONRemove, 2, -1}},
-	ast.JSONMerge:   &jsonMergeFunctionClass{baseFunctionClass{ast.JSONMerge, 2, -1}},
-	ast.JSONObject:  &jsonObjectFunctionClass{baseFunctionClass{ast.JSONObject, 0, -1}},
-	ast.JSONArray:   &jsonArrayFunctionClass{baseFunctionClass{ast.JSONArray, 0, -1}},
+	ast.JSONType:     &jsonTypeFunctionClass{baseFunctionClass{ast.JSONType, 1, 1}},
+	ast.JSONExtract:  &jsonExtractFunctionClass{baseFunctionClass{ast.JSONExtract, 2, -1}},
+	ast.JSONUnquote:  &jsonUnquoteFunctionClass{baseFunctionClass{ast.JSONUnquote, 1, 1}},
+	ast.JSONSet:      &jsonSetFunctionClass{baseFunctionClass{ast.JSONSet, 3, -1}},
+	ast.JSONInsert:   &jsonInsertFunctionClass{baseFunctionClass{ast.JSONInsert, 3, -1}},
+	ast.JSONReplace:  &jsonReplaceFunctionClass{baseFunctionClass{ast.JSONReplace, 3, -1}},
+	ast.JSONRemove:   &jsonRemoveFunctionClass{baseFunctionClass{ast.JSONRemove, 2, -1}},
+	ast.JSONMerge:    &jsonMergeFunctionClass{baseFunctionClass{ast.JSONMerge, 2, -1}},
+	ast.JSONObject:   &jsonObjectFunctionClass{baseFunctionClass{ast.JSONObject, 0, -1}},
+	ast.JSONArray:    &jsonArrayFunctionClass{baseFunctionClass{ast.JSONArray, 0, -1}},
+	ast.JSONContains: &jsonContainsFunctionClass{baseFunctionClass{ast.JSONContains, 2, 3}},
 }

expression/builtin_cast.go

@@ -968,7 +968,7 @@ func (b *builtinCastDecimalAsTimeSig) evalTime(row chunk.Row) (res types.Time, i
 		return res, isNull, errors.Trace(err)
 	}
 	sc := b.ctx.GetSessionVars().StmtCtx
-	res, err = types.ParseTime(sc, string(val.ToString()), b.tp.Tp, b.tp.Decimal)
+	res, err = types.ParseTimeFromFloatString(sc, string(val.ToString()), b.tp.Tp, b.tp.Decimal)
 	if err != nil {
 		return res, false, errors.Trace(err)
 	}

expression/builtin_json.go

@@ -35,6 +35,7 @@ var (
 	_ functionClass = &jsonMergeFunctionClass{}
 	_ functionClass = &jsonObjectFunctionClass{}
 	_ functionClass = &jsonArrayFunctionClass{}
+	_ functionClass = &jsonContainsFunctionClass{}
 
 	// Type of JSON value.
 	_ builtinFunc = &builtinJSONTypeSig{}
@@ -56,6 +57,8 @@ var (
 	_ builtinFunc = &builtinJSONRemoveSig{}
 	// Merge JSON documents, preserving duplicate keys.
 	_ builtinFunc = &builtinJSONMergeSig{}
+	// Check JSON document contains specific target.
+	_ builtinFunc = &builtinJSONContainsSig{}
 )
 
 type jsonTypeFunctionClass struct {
@@ -548,3 +551,66 @@ func jsonModify(ctx sessionctx.Context, args []Expression, row chunk.Row, mt jso
 	}
 	return res, false, nil
 }
+
+type jsonContainsFunctionClass struct {
+	baseFunctionClass
+}
+
+type builtinJSONContainsSig struct {
+	baseBuiltinFunc
+}
+
+func (b *builtinJSONContainsSig) Clone() builtinFunc {
+	newSig := &builtinJSONContainsSig{}
+	newSig.cloneFrom(&b.baseBuiltinFunc)
+	return newSig
+}
+
+func (c *jsonContainsFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) {
+	if err := c.verifyArgs(args); err != nil {
+		return nil, errors.Trace(err)
+	}
+	argTps := []types.EvalType{types.ETJson, types.ETJson}
+	if len(args) == 3 {
+		argTps = append(argTps, types.ETString)
+	}
+	bf := newBaseBuiltinFuncWithTp(ctx, args, types.ETInt, argTps...)
+	sig := &builtinJSONContainsSig{bf}
+	sig.setPbCode(tipb.ScalarFuncSig_JsonContainsSig)
+	return sig, nil
+}
+
+func (b *builtinJSONContainsSig) evalInt(row chunk.Row) (res int64, isNull bool, err error) {
+	obj, isNull, err := b.args[0].EvalJSON(b.ctx, row)
+	if isNull || err != nil {
+		return res, isNull, errors.Trace(err)
+	}
+	target, isNull, err := b.args[1].EvalJSON(b.ctx, row)
+	if isNull || err != nil {
+		return res, isNull, errors.Trace(err)
+	}
+	var pathExpr json.PathExpression
+	if len(b.args) == 3 {
+		path, isNull, err := b.args[2].EvalString(b.ctx, row)
+		if isNull || err != nil {
+			return res, isNull, errors.Trace(err)
+		}
+		pathExpr, err = json.ParseJSONPathExpr(path)
+		if err != nil {
+			return res, true, errors.Trace(err)
+		}
+		if pathExpr.ContainsAnyAsterisk() {
+			return res, true, json.ErrInvalidJSONPathWildcard
+		}
+		var exists bool
+		obj, exists = obj.Extract([]json.PathExpression{pathExpr})
+		if !exists {
+			return res, true, nil
+		}
+	}
+
+	if json.ContainsBinary(obj, target) {
+		return 1, false, nil
+	}
+	return 0, false, nil
+}

expression/builtin_json_test.go

@@ -259,7 +259,7 @@ func (s *testEvaluatorSuite) TestJSONObject(c *C) {
 	}
 }
 
-func (s *testEvaluatorSuite) TestJSONORemove(c *C) {
+func (s *testEvaluatorSuite) TestJSONRemove(c *C) {
 	defer testleak.AfterTest(c)()
 	fc := funcs[ast.JSONRemove]
 	tbl := []struct {
@@ -308,3 +308,66 @@ func (s *testEvaluatorSuite) TestJSONORemove(c *C) {
 		}
 	}
 }
+
+func (s *testEvaluatorSuite) TestJSONContains(c *C) {
+	defer testleak.AfterTest(c)()
+	fc := funcs[ast.JSONContains]
+	tbl := []struct {
+		input    []interface{}
+		expected interface{}
+		success  bool
+	}{
+		// Tests nil arguments
+		{[]interface{}{nil, `1`, "$.c"}, nil, true},
+		{[]interface{}{`{"a": [1, 2, {"aa": "xx"}]}`, nil, "$.a[3]"}, nil, true},
+		{[]interface{}{`{"a": [1, 2, {"aa": "xx"}]}`, `1`, nil}, nil, true},
+		// Tests with path expression
+		{[]interface{}{`[1,2,[1,[5,[3]]]]`, `[1,3]`, "$[2]"}, 1, true},
+		{[]interface{}{`[1,2,[1,[5,{"a":[2,3]}]]]`, `[1,{"a":[3]}]`, "$[2]"}, 1, true},
+		{[]interface{}{`[{"a":1}]`, `{"a":1}`, "$"}, 1, true},
+		{[]interface{}{`[{"a":1,"b":2}]`, `{"a":1,"b":2}`, "$"}, 1, true},
+		{[]interface{}{`[{"a":{"a":1},"b":2}]`, `{"a":1}`, "$.a"}, 0, true},
+		// Tests without path expression
+		{[]interface{}{`{}`, `{}`}, 1, true},
+		{[]interface{}{`{"a":1}`, `{}`}, 1, true},
+		{[]interface{}{`{"a":1}`, `1`}, 0, true},
+		{[]interface{}{`{"a":[1]}`, `[1]`}, 0, true},
+		{[]interface{}{`{"b":2, "c":3}`, `{"c":3}`}, 1, true},
+		{[]interface{}{`1`, `1`}, 1, true},
+		{[]interface{}{`[1]`, `1`}, 1, true},
+		{[]interface{}{`[1,2]`, `[1]`}, 1, true},
+		{[]interface{}{`[1,2]`, `[1,3]`}, 0, true},
+		{[]interface{}{`[1,2]`, `["1"]`}, 0, true},
+		{[]interface{}{`[1,2,[1,3]]`, `[1,3]`}, 1, true},
+		{[]interface{}{`[1,2,[1,[5,[3]]]]`, `[1,3]`}, 1, true},
+		{[]interface{}{`[1,2,[1,[5,{"a":[2,3]}]]]`, `[1,{"a":[3]}]`}, 1, true},
+		{[]interface{}{`[{"a":1}]`, `{"a":1}`}, 1, true},
+		{[]interface{}{`[{"a":1,"b":2}]`, `{"a":1}`}, 1, true},
+		{[]interface{}{`[{"a":{"a":1},"b":2}]`, `{"a":1}`}, 0, true},
+		// Tests path expression contains any asterisk
+		{[]interface{}{`{"a": [1, 2, {"aa": "xx"}]}`, `1`, "$.*"}, nil, false},
+		{[]interface{}{`{"a": [1, 2, {"aa": "xx"}]}`, `1`, "$[*]"}, nil, false},
+		{[]interface{}{`{"a": [1, 2, {"aa": "xx"}]}`, `1`, "$**.a"}, nil, false},
+		// Tests path expression does not identify a section of the target document
+		{[]interface{}{`{"a": [1, 2, {"aa": "xx"}]}`, `1`, "$.c"}, nil, true},
+		{[]interface{}{`{"a": [1, 2, {"aa": "xx"}]}`, `1`, "$.a[3]"}, nil, true},
+		{[]interface{}{`{"a": [1, 2, {"aa": "xx"}]}`, `1`, "$.a[2].b"}, nil, true},
+	}
+	for _, t := range tbl {
+		args := types.MakeDatums(t.input...)
+		f, err := fc.getFunction(s.ctx, s.datumsToConstants(args))
+		c.Assert(err, IsNil)
+		d, err := evalBuiltinFunc(f, chunk.Row{})
+
+		if t.success {
+			c.Assert(err, IsNil)
+			if t.expected == nil {
+				c.Assert(d.IsNull(), IsTrue)
+			} else {
+				c.Assert(d.GetInt64(), Equals, int64(t.expected.(int)))
+			}
+		} else {
+			c.Assert(err, NotNil)
+		}
+	}
+}

expression/builtin_time.go

@@ -993,7 +993,10 @@ func (b *builtinMonthSig) evalInt(row chunk.Row) (int64, bool, error) {
 	}
 
 	if date.IsZero() {
-		return 0, true, errors.Trace(handleInvalidTimeError(b.ctx, types.ErrIncorrectDatetimeValue.GenByArgs(date.String())))
+		if b.ctx.GetSessionVars().SQLMode.HasNoZeroDateMode() {
+			return 0, true, errors.Trace(handleInvalidTimeError(b.ctx, types.ErrIncorrectDatetimeValue.GenByArgs(date.String())))
+		}
+		return 0, false, nil
 	}
 
 	return int64(date.Time.Month()), false, nil
@@ -1030,9 +1033,9 @@ func (b *builtinMonthNameSig) evalString(row chunk.Row) (string, bool, error) {
 		return "", true, errors.Trace(handleInvalidTimeError(b.ctx, err))
 	}
 	mon := arg.Time.Month()
-	if arg.IsZero() || mon < 0 || mon > len(types.MonthNames) {
+	if (arg.IsZero() && b.ctx.GetSessionVars().SQLMode.HasNoZeroDateMode()) || mon < 0 || mon > len(types.MonthNames) {
 		return "", true, errors.Trace(handleInvalidTimeError(b.ctx, types.ErrIncorrectDatetimeValue.GenByArgs(arg.String())))
-	} else if mon == 0 {
+	} else if mon == 0 || arg.IsZero() {
 		return "", true, nil
 	}
 	return types.MonthNames[mon-1], false, nil
@@ -1111,7 +1114,10 @@ func (b *builtinDayOfMonthSig) evalInt(row chunk.Row) (int64, bool, error) {
 		return 0, true, errors.Trace(handleInvalidTimeError(b.ctx, err))
 	}
 	if arg.IsZero() {
-		return 0, true, errors.Trace(handleInvalidTimeError(b.ctx, types.ErrIncorrectDatetimeValue.GenByArgs(arg.String())))
+		if b.ctx.GetSessionVars().SQLMode.HasNoZeroDateMode() {
+			return 0, true, errors.Trace(handleInvalidTimeError(b.ctx, types.ErrIncorrectDatetimeValue.GenByArgs(arg.String())))
+		}
+		return 0, false, nil
 	}
 	return int64(arg.Time.Day()), false, nil
 }
@@ -1393,9 +1399,11 @@ func (b *builtinYearSig) evalInt(row chunk.Row) (int64, bool, error) {
 	}
 
 	if date.IsZero() {
-		return 0, true, errors.Trace(handleInvalidTimeError(b.ctx, types.ErrIncorrectDatetimeValue.GenByArgs(date.String())))
+		if b.ctx.GetSessionVars().SQLMode.HasNoZeroDateMode() {
+			return 0, true, errors.Trace(handleInvalidTimeError(b.ctx, types.ErrIncorrectDatetimeValue.GenByArgs(date.String())))
+		}
+		return 0, false, nil
 	}
-
 	return int64(date.Time.Year()), false, nil
 }