From f7cd79fc1db9c0528acabf87855b20a658925985 Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Wed, 17 Nov 2021 15:14:14 +0800 Subject: [PATCH 1/4] done Signed-off-by: wjhuang2016 --- executor/distsql.go | 9 ++- types/datum.go | 136 +++++++++++++++++++++++++++++++++++++++++++ util/logutil/zap_log | 1 + 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 util/logutil/zap_log diff --git a/executor/distsql.go b/executor/distsql.go index 899d2e822f176..bc9e7538c67cb 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -1119,6 +1119,13 @@ func (w *tableWorker) compareData(ctx context.Context, task *lookupTableTask, ta chk := newFirstChunk(tableReader) tblInfo := w.idxLookup.table.Meta() vals := make([]types.Datum, 0, len(w.idxTblCols)) + + // Prepare collator for compare. + collators := make([]collate.Collator, 0, len(w.idxColTps)) + for _, tp := range w.idxColTps { + collators = append(collators, collate.GetCollator(tp.Collate)) + } + for { err := Next(ctx, tableReader, chk) if err != nil { @@ -1160,7 +1167,7 @@ func (w *tableWorker) compareData(ctx context.Context, task *lookupTableTask, ta tp := &col.FieldType idxVal := idxRow.GetDatum(i, tp) tablecodec.TruncateIndexValue(&idxVal, w.idxLookup.index.Columns[i], col.ColumnInfo) - cmpRes, err := idxVal.CompareDatum(sctx, &val) + cmpRes, err := idxVal.CompareDatumNew(sctx, &val, collators[i]) if err != nil { return ErrDataInConsistentMisMatchIndex.GenWithStackByArgs(col.Name, handle, idxRow.GetDatum(i, tp), val, err) diff --git a/types/datum.go b/types/datum.go index d79a086cd7346..e54dcfb2e160b 100644 --- a/types/datum.go +++ b/types/datum.go @@ -31,6 +31,7 @@ import ( "github.com/pingcap/tidb/parser/types" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/types/json" + "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/hack" ) @@ -548,7 +549,63 @@ func (d *Datum) SetValue(val interface{}, tp *types.FieldType) { } } +// CompareDatumNew compares datum to another datum. +// Notes: don't rely on datum.collation to get the collator, it's tend to buggy. +// TODO: use this function to replace CompareDatum. After we remove all of usage of CompareDatum, we can rename this function back to CompareDatum. +func (d *Datum) CompareDatumNew(sc *stmtctx.StatementContext, ad *Datum, comparer collate.Collator) (int, error) { + if d.k == KindMysqlJSON && ad.k != KindMysqlJSON { + cmp, err := ad.CompareDatumNew(sc, d, comparer) + return cmp * -1, errors.Trace(err) + } + switch ad.k { + case KindNull: + if d.k == KindNull { + return 0, nil + } + return 1, nil + case KindMinNotNull: + if d.k == KindNull { + return -1, nil + } else if d.k == KindMinNotNull { + return 0, nil + } + return 1, nil + case KindMaxValue: + if d.k == KindMaxValue { + return 0, nil + } + return -1, nil + case KindInt64: + return d.compareInt64(sc, ad.GetInt64()) + case KindUint64: + return d.compareUint64(sc, ad.GetUint64()) + case KindFloat32, KindFloat64: + return d.compareFloat64(sc, ad.GetFloat64()) + case KindString: + return d.compareStringNew(sc, ad.GetString(), comparer) + case KindBytes: + return comparer.Compare(d.GetString(), ad.GetString()), nil + case KindMysqlDecimal: + return d.compareMysqlDecimal(sc, ad.GetMysqlDecimal()) + case KindMysqlDuration: + return d.compareMysqlDuration(sc, ad.GetMysqlDuration()) + case KindMysqlEnum: + return d.compareMysqlEnumNew(sc, ad.GetMysqlEnum(), comparer) + case KindBinaryLiteral, KindMysqlBit: + return d.compareBinaryLiteralNew(sc, ad.GetBinaryLiteral4Cmp(), comparer) + case KindMysqlSet: + return d.compareMysqlSetNew(sc, ad.GetMysqlSet(), comparer) + case KindMysqlJSON: + return d.compareMysqlJSON(sc, ad.GetMysqlJSON()) + case KindMysqlTime: + return d.compareMysqlTime(sc, ad.GetMysqlTime()) + default: + return 0, nil + } +} + // CompareDatum compares datum to another datum. +// Deprecated: will be replaced with CompareDatumNew. // TODO: return error properly. func (d *Datum) CompareDatum(sc *stmtctx.StatementContext, ad *Datum) (int, error) { if d.k == KindMysqlJSON && ad.k != KindMysqlJSON { @@ -673,6 +730,39 @@ func (d *Datum) compareFloat64(sc *stmtctx.StatementContext, f float64) (int, er } } +func (d *Datum) compareStringNew(sc *stmtctx.StatementContext, s string, comparer collate.Collator) (int, error) { + switch d.k { + case KindNull, KindMinNotNull: + return -1, nil + case KindMaxValue: + return 1, nil + case KindString, KindBytes: + return comparer.Compare(d.GetString(), s), nil + case KindMysqlDecimal: + dec := new(MyDecimal) + err := sc.HandleTruncate(dec.FromString(hack.Slice(s))) + return d.GetMysqlDecimal().Compare(dec), errors.Trace(err) + case KindMysqlTime: + dt, err := ParseDatetime(sc, s) + return d.GetMysqlTime().Compare(dt), errors.Trace(err) + case KindMysqlDuration: + dur, err := ParseDuration(sc, s, MaxFsp) + return d.GetMysqlDuration().Compare(dur), errors.Trace(err) + case KindMysqlSet: + return comparer.Compare(d.GetMysqlSet().String(), s), nil + case KindMysqlEnum: + return comparer.Compare(d.GetMysqlEnum().String(), s), nil + case KindBinaryLiteral, KindMysqlBit: + return comparer.Compare(d.GetBinaryLiteral4Cmp().String(), s), nil + default: + fVal, err := StrToFloat(sc, s, false) + if err != nil { + return 0, errors.Trace(err) + } + return d.compareFloat64(sc, fVal) + } +} + func (d *Datum) compareString(sc *stmtctx.StatementContext, s string, retCollation string) (int, error) { switch d.k { case KindNull, KindMinNotNull: @@ -748,6 +838,52 @@ func (d *Datum) compareMysqlDuration(sc *stmtctx.StatementContext, dur Duration) } } +func (d *Datum) compareMysqlEnumNew(sc *stmtctx.StatementContext, enum Enum, comparer collate.Collator) (int, error) { + switch d.k { + case KindNull, KindMinNotNull: + return -1, nil + case KindMaxValue: + return 1, nil + case KindString, KindBytes, KindMysqlEnum, KindMysqlSet: + return comparer.Compare(d.GetString(), enum.String()), nil + default: + return d.compareFloat64(sc, enum.ToNumber()) + } +} + +func (d *Datum) compareBinaryLiteralNew(sc *stmtctx.StatementContext, b BinaryLiteral, comparer collate.Collator) (int, error) { + switch d.k { + case KindNull, KindMinNotNull: + return -1, nil + case KindMaxValue: + return 1, nil + case KindString, KindBytes: + fallthrough // in this case, d is converted to Binary and then compared with b + case KindBinaryLiteral, KindMysqlBit: + return comparer.Compare(d.GetBinaryLiteral4Cmp().ToString(), b.ToString()), nil + default: + val, err := b.ToInt(sc) + if err != nil { + return 0, errors.Trace(err) + } + result, err := d.compareFloat64(sc, float64(val)) + return result, errors.Trace(err) + } +} + +func (d *Datum) compareMysqlSetNew(sc *stmtctx.StatementContext, set Set, comparer collate.Collator) (int, error) { + switch d.k { + case KindNull, KindMinNotNull: + return -1, nil + case KindMaxValue: + return 1, nil + case KindString, KindBytes, KindMysqlEnum, KindMysqlSet: + return comparer.Compare(d.GetString(), set.String()), nil + default: + return d.compareFloat64(sc, set.ToNumber()) + } +} + func (d *Datum) compareMysqlEnum(sc *stmtctx.StatementContext, enum Enum) (int, error) { switch d.k { case KindNull, KindMinNotNull: diff --git a/util/logutil/zap_log b/util/logutil/zap_log new file mode 100644 index 0000000000000..5e559b382ff37 --- /dev/null +++ b/util/logutil/zap_log @@ -0,0 +1 @@ +[2021/11/17 14:10:11.580 +08:00] [INFO] [log.go:165] ["replaced global logger with config: {\"level\":\"info\",\"format\":\"text\",\"disable-timestamp\":false,\"file\":{\"filename\":\"zap_log\",\"max-size\":4096,\"max-days\":14,\"max-backups\":0},\"development\":false,\"disable-caller\":false,\"disable-stacktrace\":false,\"disable-error-verbose\":false,\"sampling\":null}"] From c8e4165d608e90689a0112811429fecadcbea0cf Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Wed, 17 Nov 2021 15:14:34 +0800 Subject: [PATCH 2/4] fix Signed-off-by: wjhuang2016 --- util/logutil/zap_log | 1 - 1 file changed, 1 deletion(-) delete mode 100644 util/logutil/zap_log diff --git a/util/logutil/zap_log b/util/logutil/zap_log deleted file mode 100644 index 5e559b382ff37..0000000000000 --- a/util/logutil/zap_log +++ /dev/null @@ -1 +0,0 @@ -[2021/11/17 14:10:11.580 +08:00] [INFO] [log.go:165] ["replaced global logger with config: {\"level\":\"info\",\"format\":\"text\",\"disable-timestamp\":false,\"file\":{\"filename\":\"zap_log\",\"max-size\":4096,\"max-days\":14,\"max-backups\":0},\"development\":false,\"disable-caller\":false,\"disable-stacktrace\":false,\"disable-error-verbose\":false,\"sampling\":null}"] From 69eb3600a7c5b4eb4170493b4bf1ba22e7e120fb Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Thu, 18 Nov 2021 10:53:49 +0800 Subject: [PATCH 3/4] rename Signed-off-by: wjhuang2016 --- executor/distsql.go | 2 +- types/datum.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/executor/distsql.go b/executor/distsql.go index bc9e7538c67cb..1014e5cba3eee 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -1167,7 +1167,7 @@ func (w *tableWorker) compareData(ctx context.Context, task *lookupTableTask, ta tp := &col.FieldType idxVal := idxRow.GetDatum(i, tp) tablecodec.TruncateIndexValue(&idxVal, w.idxLookup.index.Columns[i], col.ColumnInfo) - cmpRes, err := idxVal.CompareDatumNew(sctx, &val, collators[i]) + cmpRes, err := idxVal.Compare(sctx, &val, collators[i]) if err != nil { return ErrDataInConsistentMisMatchIndex.GenWithStackByArgs(col.Name, handle, idxRow.GetDatum(i, tp), val, err) diff --git a/types/datum.go b/types/datum.go index e54dcfb2e160b..9674f6148a8ab 100644 --- a/types/datum.go +++ b/types/datum.go @@ -549,12 +549,12 @@ func (d *Datum) SetValue(val interface{}, tp *types.FieldType) { } } -// CompareDatumNew compares datum to another datum. +// Compare compares datum to another datum. // Notes: don't rely on datum.collation to get the collator, it's tend to buggy. // TODO: use this function to replace CompareDatum. After we remove all of usage of CompareDatum, we can rename this function back to CompareDatum. -func (d *Datum) CompareDatumNew(sc *stmtctx.StatementContext, ad *Datum, comparer collate.Collator) (int, error) { +func (d *Datum) Compare(sc *stmtctx.StatementContext, ad *Datum, comparer collate.Collator) (int, error) { if d.k == KindMysqlJSON && ad.k != KindMysqlJSON { - cmp, err := ad.CompareDatumNew(sc, d, comparer) + cmp, err := ad.Compare(sc, d, comparer) return cmp * -1, errors.Trace(err) } switch ad.k { @@ -605,7 +605,7 @@ func (d *Datum) CompareDatumNew(sc *stmtctx.StatementContext, ad *Datum, compare } // CompareDatum compares datum to another datum. -// Deprecated: will be replaced with CompareDatumNew. +// Deprecated: will be replaced with Compare. // TODO: return error properly. func (d *Datum) CompareDatum(sc *stmtctx.StatementContext, ad *Datum) (int, error) { if d.k == KindMysqlJSON && ad.k != KindMysqlJSON { From 43590d599b1078414a70ee633eb1fa283f0e6a9f Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Mon, 22 Nov 2021 15:47:24 +0800 Subject: [PATCH 4/4] fix Signed-off-by: wjhuang2016 --- executor/distsql.go | 1 - 1 file changed, 1 deletion(-) diff --git a/executor/distsql.go b/executor/distsql.go index 5323e9bbc278c..3edb1dd709168 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -1176,7 +1176,6 @@ func (w *tableWorker) compareData(ctx context.Context, task *lookupTableTask, ta tp := &col.FieldType idxVal := idxRow.GetDatum(i, tp) tablecodec.TruncateIndexValue(&idxVal, w.idxLookup.index.Columns[i], col.ColumnInfo) - cmpRes, err := idxVal.Compare(sctx, &val, collators[i]) cmpRes, err := idxVal.Compare(sctx, &vals[i], collators[i]) if err != nil { return ErrDataInConsistentMisMatchIndex.GenWithStackByArgs(col.Name,