From 25f9ee4699f5a669c031fb8508bb7c2ad2e41706 Mon Sep 17 00:00:00 2001 From: ciscoxll Date: Thu, 3 Jan 2019 16:20:26 +0800 Subject: [PATCH 1/4] modify different character sets varchar type column maximum length limit --- ddl/db_test.go | 17 +++++++++++++++ ddl/ddl_api.go | 44 ++++++++++++++++++++++++++++++++++++++ planner/core/preprocess.go | 17 +++++---------- 3 files changed, 66 insertions(+), 12 deletions(-) diff --git a/ddl/db_test.go b/ddl/db_test.go index 377b0061aa372..7cd13278fd98f 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -1945,6 +1945,23 @@ func (s *testDBSuite) TestColumnModifyingDefinition(c *C) { s.testErrorCode(c, "alter table test2 change c1 a1 bigint not null;", tmysql.WarnDataTruncated) } +func (s *testDBSuite) TestCheckTooBigFieldLength(c *C) { + s.tk = testkit.NewTestKit(c, s.store) + s.tk.MustExec("use test") + s.tk.MustExec("drop table if exists tr_01;") + s.tk.MustExec("create table tr_01 (id int, name varchar(20000), purchased date ) default charset=utf8 collate=utf8_bin;") + + s.tk.MustExec("drop table if exists tr_02;") + s.tk.MustExec("create table tr_02 (id int, name varchar(16000), purchased date ) default charset=utf8mb4 collate=utf8mb4_bin;") + + s.tk.MustExec("drop table if exists tr_03;") + s.tk.MustExec("create table tr_03 (id int, name varchar(65534), purchased date ) default charset=latin1;") + + s.testErrorCode(c, "create table tr (id int, name varchar(30000), purchased date ) default charset=utf8 collate=utf8_bin;", tmysql.ErrTooBigFieldlength) + s.testErrorCode(c, "create table tr (id int, name varchar(20000) charset utf8mb4, purchased date ) default charset=utf8 collate=utf8;", tmysql.ErrTooBigFieldlength) + s.testErrorCode(c, "create table tr (id int, name varchar(65536), purchased date ) default charset=latin1;", tmysql.ErrTooBigFieldlength) +} + func (s *testDBSuite) TestModifyColumnRollBack(c *C) { s.tk = testkit.NewTestKit(c, s.store) s.mustExec(c, "use test_db") diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 2526eb1b6ea49..43ba8382de335 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -658,6 +658,41 @@ func checkColumnsAttributes(colDefs []*ast.ColumnDef) error { return nil } +// checkColumnFieldLength check the maximum length limit for different character set varchar type columns. +func checkColumnFieldLength(schema *model.DBInfo, colDefs []*ast.ColumnDef, tbInfo *model.TableInfo) error { + for _, colDef := range colDefs { + if colDef.Tp.Tp == mysql.TypeVarchar { + var setCharset string + setCharset = mysql.DefaultCharset + if len(schema.Charset) != 0 { + setCharset = schema.Charset + } + if len(tbInfo.Charset) != 0 { + setCharset = tbInfo.Charset + } + + err := IsTooBigFieldLength(colDef, setCharset) + if err != nil { + return errors.Trace(err) + } + } + } + return nil +} + +func IsTooBigFieldLength(colDef *ast.ColumnDef, setCharset string) error { + desc, err := charset.GetCharsetDesc(setCharset) + if err != nil { + return errors.Trace(err) + } + maxFlen := mysql.MaxFieldVarCharLength + maxFlen /= desc.Maxlen + if colDef.Tp.Flen != types.UnspecifiedLength && colDef.Tp.Flen > maxFlen { + return types.ErrTooBigFieldLength.GenWithStack("Column length too big for column '%s' (max = %d); use BLOB or TEXT instead", colDef.Name.Name.O, maxFlen) + } + return errors.Trace(err) +} + // checkColumnAttributes check attributes for single column. func checkColumnAttributes(colName string, tp *types.FieldType) error { switch tp.Tp { @@ -1007,6 +1042,10 @@ func (d *ddl) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) (err e if err != nil { return errors.Trace(err) } + if err = checkColumnFieldLength(schema, s.Cols, tbInfo); err != nil { + return errors.Trace(err) + } + err = d.doDDLJob(ctx, job) if err == nil { if tbInfo.AutoIncID > 1 { @@ -1982,6 +2021,11 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or // `modifyColumnTp` indicates that there is a type modification. modifyColumnTp = mysql.TypeNull } + + if err = checkColumnFieldLength(schema, spec.NewColumns, t.Meta()); err != nil { + return nil, errors.Trace(err) + } + // As same with MySQL, we don't support modifying the stored status for generated columns. if err = checkModifyGeneratedColumn(t.Cols(), col, newCol); err != nil { return nil, errors.Trace(err) diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index a89d058ddcb94..9a5c974dfdc0d 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -20,7 +20,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/charset" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/ddl" @@ -489,22 +488,16 @@ func checkColumn(colDef *ast.ColumnDef) error { return types.ErrTooBigFieldLength.GenWithStack("Column length too big for column '%s' (max = %d); use BLOB or TEXT instead", colDef.Name.Name.O, mysql.MaxFieldCharLength) } case mysql.TypeVarchar: - maxFlen := mysql.MaxFieldVarCharLength - cs := tp.Charset - // TODO: TableDefaultCharset-->DatabaseDefaultCharset-->SystemDefaultCharset. - // TODO: Change TableOption parser to parse collate. - // Reference https://github.com/pingcap/tidb/blob/b091e828cfa1d506b014345fb8337e424a4ab905/ddl/ddl_api.go#L185-L204 + // Here logical move backend processing, need to get all levels of character set to set the character set of column. + // It's not easy to get the schema charset and table charset here. + // ColumnDefaultCharset --> TableDefaultCharset-->DatabaseDefaultCharset-->SystemDefaultCharset. if len(tp.Charset) == 0 { - cs = mysql.DefaultCharset + return nil } - desc, err := charset.GetCharsetDesc(cs) + err := ddl.IsTooBigFieldLength(colDef, tp.Charset) if err != nil { return errors.Trace(err) } - maxFlen /= desc.Maxlen - if tp.Flen != types.UnspecifiedLength && tp.Flen > maxFlen { - return types.ErrTooBigFieldLength.GenWithStack("Column length too big for column '%s' (max = %d); use BLOB or TEXT instead", colDef.Name.Name.O, maxFlen) - } case mysql.TypeFloat, mysql.TypeDouble: if tp.Decimal > mysql.MaxFloatingTypeScale { return types.ErrTooBigScale.GenWithStackByArgs(tp.Decimal, colDef.Name.Name.O, mysql.MaxFloatingTypeScale) From 5ae6150dcfb3a529bbc548e1a3dd199bf24ba76e Mon Sep 17 00:00:00 2001 From: ciscoxll Date: Thu, 3 Jan 2019 16:32:47 +0800 Subject: [PATCH 2/4] Fix CI --- ddl/ddl_api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 43ba8382de335..fc2f9cbf15fa8 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -680,6 +680,7 @@ func checkColumnFieldLength(schema *model.DBInfo, colDefs []*ast.ColumnDef, tbIn return nil } +// IsTooBigFieldLength check if the varchar type column exceeds the maximum length limit. func IsTooBigFieldLength(colDef *ast.ColumnDef, setCharset string) error { desc, err := charset.GetCharsetDesc(setCharset) if err != nil { From 62913a8840b06418c24bd931856a0f1584df2738 Mon Sep 17 00:00:00 2001 From: ciscoxll Date: Wed, 9 Jan 2019 20:20:58 +0800 Subject: [PATCH 3/4] Address comment --- ddl/db_test.go | 3 +++ ddl/ddl_api.go | 16 +++++++++++----- planner/core/preprocess.go | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/ddl/db_test.go b/ddl/db_test.go index 7cd13278fd98f..baba39a102598 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -1957,6 +1957,9 @@ func (s *testDBSuite) TestCheckTooBigFieldLength(c *C) { s.tk.MustExec("drop table if exists tr_03;") s.tk.MustExec("create table tr_03 (id int, name varchar(65534), purchased date ) default charset=latin1;") + s.tk.MustExec("drop table if exists tr_04;") + s.tk.MustExec("create table tr_04 (a varchar(20000) ) default charset utf8;") + s.testErrorCode(c, "alter table tr_04 convert to character set utf8mb4;", tmysql.ErrTooBigFieldlength) s.testErrorCode(c, "create table tr (id int, name varchar(30000), purchased date ) default charset=utf8 collate=utf8_bin;", tmysql.ErrTooBigFieldlength) s.testErrorCode(c, "create table tr (id int, name varchar(20000) charset utf8mb4, purchased date ) default charset=utf8 collate=utf8;", tmysql.ErrTooBigFieldlength) s.testErrorCode(c, "create table tr (id int, name varchar(65536), purchased date ) default charset=latin1;", tmysql.ErrTooBigFieldlength) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index fc2f9cbf15fa8..0ddbf78c64efa 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -671,7 +671,7 @@ func checkColumnFieldLength(schema *model.DBInfo, colDefs []*ast.ColumnDef, tbIn setCharset = tbInfo.Charset } - err := IsTooBigFieldLength(colDef, setCharset) + err := IsTooBigFieldLength(colDef.Tp.Flen, colDef.Name.Name.O, setCharset) if err != nil { return errors.Trace(err) } @@ -681,17 +681,17 @@ func checkColumnFieldLength(schema *model.DBInfo, colDefs []*ast.ColumnDef, tbIn } // IsTooBigFieldLength check if the varchar type column exceeds the maximum length limit. -func IsTooBigFieldLength(colDef *ast.ColumnDef, setCharset string) error { +func IsTooBigFieldLength(colDefTpFlen int, colDefName, setCharset string) error { desc, err := charset.GetCharsetDesc(setCharset) if err != nil { return errors.Trace(err) } maxFlen := mysql.MaxFieldVarCharLength maxFlen /= desc.Maxlen - if colDef.Tp.Flen != types.UnspecifiedLength && colDef.Tp.Flen > maxFlen { - return types.ErrTooBigFieldLength.GenWithStack("Column length too big for column '%s' (max = %d); use BLOB or TEXT instead", colDef.Name.Name.O, maxFlen) + if colDefTpFlen != types.UnspecifiedLength && colDefTpFlen > maxFlen { + return types.ErrTooBigFieldLength.GenWithStack("Column length too big for column '%s' (max = %d); use BLOB or TEXT instead", colDefName, maxFlen) } - return errors.Trace(err) + return nil } // checkColumnAttributes check attributes for single column. @@ -2207,6 +2207,12 @@ func (d *ddl) AlterTableCharsetAndCollate(ctx sessionctx.Context, ident ast.Iden return errors.Trace(err) } + for _, col := range tb.Meta().Cols() { + if col.Tp == mysql.TypeVarchar { + err = IsTooBigFieldLength(col.Flen, col.Name.O, toCharset) + return errors.Trace(err) + } + } job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index 9a5c974dfdc0d..c8084c5edcb20 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -494,7 +494,7 @@ func checkColumn(colDef *ast.ColumnDef) error { if len(tp.Charset) == 0 { return nil } - err := ddl.IsTooBigFieldLength(colDef, tp.Charset) + err := ddl.IsTooBigFieldLength(colDef.Tp.Flen, colDef.Name.Name.O, tp.Charset) if err != nil { return errors.Trace(err) } From b26e36739b9363fe541aaf3c019e4827747019b9 Mon Sep 17 00:00:00 2001 From: ciscoxll Date: Mon, 14 Jan 2019 16:44:32 +0800 Subject: [PATCH 4/4] Address comment --- planner/core/preprocess.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index c8084c5edcb20..a4ed20acd77eb 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -488,10 +488,10 @@ func checkColumn(colDef *ast.ColumnDef) error { return types.ErrTooBigFieldLength.GenWithStack("Column length too big for column '%s' (max = %d); use BLOB or TEXT instead", colDef.Name.Name.O, mysql.MaxFieldCharLength) } case mysql.TypeVarchar: - // Here logical move backend processing, need to get all levels of character set to set the character set of column. - // It's not easy to get the schema charset and table charset here. - // ColumnDefaultCharset --> TableDefaultCharset-->DatabaseDefaultCharset-->SystemDefaultCharset. if len(tp.Charset) == 0 { + // It's not easy to get the schema charset and table charset here. + // The charset is determined by the order ColumnDefaultCharset --> TableDefaultCharset-->DatabaseDefaultCharset-->SystemDefaultCharset. + // return nil, to make the check in the ddl.CreateTable. return nil } err := ddl.IsTooBigFieldLength(colDef.Tp.Flen, colDef.Name.Name.O, tp.Charset)