This is an automated cherry-pick of pingcap#38391

Signed-off-by: ti-chi-bot <[email protected]>
ti-chi-bot · Oct 17, 2022 · e6a0c2e · e6a0c2e
1 parent 1acd0b6
commit e6a0c2e
Show file tree

Hide file tree

Showing 3 changed files with 614 additions and 3 deletions.
diff --git a/br/pkg/lightning/mydump/parquet_parser.go b/br/pkg/lightning/mydump/parquet_parser.go
@@ -430,7 +430,16 @@ func setDatumByString(d *types.Datum, v string, meta *parquet.SchemaElement) {
 	if meta.LogicalType != nil && meta.LogicalType.DECIMAL != nil {
 		v = binaryToDecimalStr([]byte(v), int(meta.LogicalType.DECIMAL.Scale))
 	}
+<<<<<<< HEAD
 	d.SetString(v, "")
+=======
+	if meta.Type != nil && *meta.Type == parquet.Type_INT96 && len(v) == 96/8 {
+		ts := int96ToTime([]byte(v))
+		ts = ts.UTC()
+		v = ts.Format(utcTimeLayout)
+	}
+	d.SetString(v, "utf8mb4_bin")
+>>>>>>> 68305e9004 (lightning: specify collation when parquet value to string datum (#38391))
 }
 
 func binaryToDecimalStr(rawBytes []byte, scale int) string {
@@ -487,20 +496,26 @@ func setDatumByInt(d *types.Datum, v int64, meta *parquet.SchemaElement) error {
 		}
 		val := fmt.Sprintf("%0*d", minLen, v)
 		dotIndex := len(val) - int(*meta.Scale)
-		d.SetString(val[:dotIndex]+"."+val[dotIndex:], "")
+		d.SetString(val[:dotIndex]+"."+val[dotIndex:], "utf8mb4_bin")
 	case logicalType.DATE != nil:
 		dateStr := time.Unix(v*86400, 0).Format("2006-01-02")
-		d.SetString(dateStr, "")
+		d.SetString(dateStr, "utf8mb4_bin")
 	case logicalType.TIMESTAMP != nil:
 		// convert all timestamp types (datetime/timestamp) to string
+<<<<<<< HEAD
 		timeStr := formatTime(v, logicalType.TIMESTAMP.Unit, "2006-01-02 15:04:05.999999",
 			"2006-01-02 15:04:05.999999Z", logicalType.TIMESTAMP.IsAdjustedToUTC)
 		d.SetString(timeStr, "")
+=======
+		timeStr := formatTime(v, logicalType.TIMESTAMP.Unit, timeLayout,
+			utcTimeLayout, logicalType.TIMESTAMP.IsAdjustedToUTC)
+		d.SetString(timeStr, "utf8mb4_bin")
+>>>>>>> 68305e9004 (lightning: specify collation when parquet value to string datum (#38391))
 	case logicalType.TIME != nil:
 		// convert all timestamp types (datetime/timestamp) to string
 		timeStr := formatTime(v, logicalType.TIME.Unit, "15:04:05.999999", "15:04:05.999999Z",
 			logicalType.TIME.IsAdjustedToUTC)
-		d.SetString(timeStr, "")
+		d.SetString(timeStr, "utf8mb4_bin")
 	default:
 		d.SetInt64(v)
 	}

diff --git a/br/pkg/lightning/mydump/parquet_parser_test.go b/br/pkg/lightning/mydump/parquet_parser_test.go
@@ -53,10 +53,17 @@ func (s testParquetParserSuite) TestParquetParser(c *C) {
 	c.Assert(reader.Columns(), DeepEquals, []string{"ss", "a_a"})
 
 	verifyRow := func(i int) {
+<<<<<<< HEAD
 		c.Assert(reader.lastRow.RowID, Equals, int64(i+1))
 		c.Assert(len(reader.lastRow.Row), Equals, 2)
 		c.Assert(reader.lastRow.Row[0], DeepEquals, types.NewCollationStringDatum(strconv.Itoa(i), ""))
 		c.Assert(reader.lastRow.Row[1], DeepEquals, types.NewIntDatum(int64(i)))
+=======
+		require.Equal(t, int64(i+1), reader.lastRow.RowID)
+		require.Len(t, reader.lastRow.Row, 2)
+		require.Equal(t, types.NewCollationStringDatum(strconv.Itoa(i), "utf8mb4_bin"), reader.lastRow.Row[0])
+		require.Equal(t, types.NewIntDatum(int64(i)), reader.lastRow.Row[1])
+>>>>>>> 68305e9004 (lightning: specify collation when parquet value to string datum (#38391))
 	}
 
 	// test read some rows