From ffe6c725c0d97e0436a757bcb5f666da62761bd6 Mon Sep 17 00:00:00 2001 From: Semyon Danilov Date: Fri, 15 Mar 2024 14:28:10 +0400 Subject: [PATCH 1/2] YDB-2757 Fix UUID column export (#2789) Fixes #2757 --- ydb/core/io_formats/cell_maker/cell_maker.cpp | 27 +++++++ ydb/core/tx/datashard/export_common.cpp | 10 +++ ydb/core/tx/datashard/export_common.h | 1 + .../tx/datashard/export_s3_buffer_raw.cpp | 3 + .../tx/schemeshard/ut_backup/ut_backup.cpp | 24 +++++++ .../tx/schemeshard/ut_restore/ut_restore.cpp | 71 ++++++++++++++++++- 6 files changed, 135 insertions(+), 1 deletion(-) diff --git a/ydb/core/io_formats/cell_maker/cell_maker.cpp b/ydb/core/io_formats/cell_maker/cell_maker.cpp index 1fc259159728..bf7fe1a35a88 100644 --- a/ydb/core/io_formats/cell_maker/cell_maker.cpp +++ b/ydb/core/io_formats/cell_maker/cell_maker.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -132,6 +133,21 @@ namespace { return true; } + struct TUuidHolder { + union { + ui16 Array[8]; + ui64 Halves[2]; + } Buf; + }; + + template <> + bool TryParse(TStringBuf value, TUuidHolder& result) { + if (!NUuid::ParseUuidToArray(value, result.Buf.Array, false)) { + return false; + } + return true; + } + template using TConverter = std::function; @@ -171,6 +187,14 @@ namespace { return v.Str; } + TStringBuf UuidToStringBuf(const TUuidHolder& uuid) { + char uuidBuf[16]; + + NUuid::UuidHalfsToBytes(uuidBuf, 16, uuid.Buf.Halves[1], uuid.Buf.Halves[0]); + + return TStringBuf(uuidBuf, 16); + } + template struct TCellMaker { static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TString& err, TConverter conv = &Implicit) { @@ -297,6 +321,8 @@ bool MakeCell(TCell& cell, TStringBuf value, NScheme::TTypeInfo type, TMemoryPoo return TCellMaker>::Make(cell, value, pool, err, &Int128ToPair); case NScheme::NTypeIds::Pg: return TCellMaker::Make(cell, value, pool, err, &PgToStringBuf, type.GetTypeDesc()); + case NScheme::NTypeIds::Uuid: + return TCellMaker::Make(cell, value, pool, err, &UuidToStringBuf); default: return false; } @@ -390,6 +416,7 @@ bool CheckCellValue(const TCell& cell, NScheme::TTypeInfo type) { case NScheme::NTypeIds::JsonDocument: // checked at parsing time case NScheme::NTypeIds::DyNumber: // checked at parsing time case NScheme::NTypeIds::Pg: // checked at parsing time + case NScheme::NTypeIds::Uuid: // checked at parsing time return true; case NScheme::NTypeIds::Date: return cell.AsValue() < NUdf::MAX_DATE; diff --git a/ydb/core/tx/datashard/export_common.cpp b/ydb/core/tx/datashard/export_common.cpp index 5675d7de11e4..fa7a0fb28074 100644 --- a/ydb/core/tx/datashard/export_common.cpp +++ b/ydb/core/tx/datashard/export_common.cpp @@ -118,5 +118,15 @@ bool PgToStream(TStringBuf data, void* typeDesc, IOutputStream& out, TString& er return true; } +bool UuidToStream(const std::pair& loHi, IOutputStream& out, TString& err) { + Y_UNUSED(err); + + NYdb::TUuidValue uuid(loHi.first, loHi.second); + + out << uuid.ToString(); + + return true; +} + } // NDataShard } // NKikimr diff --git a/ydb/core/tx/datashard/export_common.h b/ydb/core/tx/datashard/export_common.h index 631e8c71e1b9..3fad32a0a305 100644 --- a/ydb/core/tx/datashard/export_common.h +++ b/ydb/core/tx/datashard/export_common.h @@ -40,6 +40,7 @@ TString DyNumberToString(TStringBuf data); bool DecimalToStream(const std::pair& loHi, IOutputStream& out, TString& err); bool DyNumberToStream(TStringBuf data, IOutputStream& out, TString& err); bool PgToStream(TStringBuf data, void* typeDesc, IOutputStream& out, TString& err); +bool UuidToStream(const std::pair& loHi, IOutputStream& out, TString& err); } // NDataShard } // NKikimr diff --git a/ydb/core/tx/datashard/export_s3_buffer_raw.cpp b/ydb/core/tx/datashard/export_s3_buffer_raw.cpp index 8e4bceadbbf8..649c46735eff 100644 --- a/ydb/core/tx/datashard/export_s3_buffer_raw.cpp +++ b/ydb/core/tx/datashard/export_s3_buffer_raw.cpp @@ -125,6 +125,9 @@ bool TS3BufferRaw::Collect(const NTable::IScan::TRow& row, IOutputStream& out) { case NScheme::NTypeIds::Pg: serialized = PgToStream(cell.AsBuf(), column.Type.GetTypeDesc(), out, ErrorString); break; + case NScheme::NTypeIds::Uuid: + serialized = UuidToStream(cell.AsValue>(), out, ErrorString); + break; default: Y_ABORT("Unsupported type"); } diff --git a/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp b/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp index 0327648b86d3..1338f2d8f42d 100644 --- a/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp +++ b/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp @@ -100,6 +100,30 @@ Y_UNIT_TEST_SUITE(TBackupTests) { }); } + Y_UNIT_TEST_WITH_COMPRESSION(BackupUuidColumn) { + TTestBasicRuntime runtime; + + Backup(runtime, ToString(Codec), R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Uuid" } + KeyColumnNames: ["key"] + )", [](TTestBasicRuntime& runtime) { + NKikimrMiniKQL::TResult result; + TString error; + NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, TTestTxConfig::FakeHiveTablets, Sprintf(R"( + ( + (let key '( '('key (Uint32 '%d) ) ) ) + (let row '( '('value (Uuid '"%s") ) ) ) + (return (AsList (UpdateRow '__user__%s key row) )) + ) + )", 1, "0000111122223333", "Table"), result, error); + + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); + UNIT_ASSERT_VALUES_EQUAL(error, ""); + }); + } + template void ShouldSucceedOnLargeData(ui32 minWriteBatchSize, const std::pair& expectedResult) { TTestBasicRuntime runtime; diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 313dc72ffb25..df9df758e647 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -1135,7 +1135,76 @@ value { env.TestWaitNotification(runtime, txId); TestGetExport(runtime, txId, "/MyRoot"); - TestImport(runtime, txId, "/MyRoot", Sprintf(R"( + TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "Backup1" + destination_path: "/MyRoot/Restored" + } + } + )", port)); + env.TestWaitNotification(runtime, txId); + TestGetImport(runtime, txId, "/MyRoot"); + } + + Y_UNIT_TEST(ExportImportUuid) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableTablePgTypes(true)); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Uuid" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + { + TString tablePath = "/MyRoot/Table"; + int partitionIdx = 0; + + auto tableDesc = DescribePath(runtime, tablePath, true, true); + const auto& tablePartitions = tableDesc.GetPathDescription().GetTablePartitions(); + UNIT_ASSERT(partitionIdx < tablePartitions.size()); + const ui64 datashardTabletId = tablePartitions[partitionIdx].GetDatashardId(); + + NKikimrMiniKQL::TResult result; + TString error; + NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, datashardTabletId, Sprintf(R"( + ( + (let key '( '('key (Uint32 '%d) ) ) ) + (let row '( '('value (Uuid '"%s") ) ) ) + (return (AsList (UpdateRow '__user__%s key row) )) + ) + )", 1, "0123456789012345", "Table"), result, error); + + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); + UNIT_ASSERT_VALUES_EQUAL(error, ""); + } + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock({}, TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/Table" + destination_prefix: "Backup1" + } + } + )", port)); + env.TestWaitNotification(runtime, txId); + TestGetExport(runtime, txId, "/MyRoot"); + + TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( ImportFromS3Settings { endpoint: "localhost:%d" scheme: HTTP From de553bce99f50dffd2b8b45e9794220ed904b718 Mon Sep 17 00:00:00 2001 From: Semyon Danilov Date: Fri, 22 Mar 2024 15:22:25 +0400 Subject: [PATCH 2/2] YDB-3080 Fix UUID prefix in select (#3030) Fixes #3080 --- ydb/core/kqp/opt/kqp_query_plan.cpp | 6 ++ ydb/core/kqp/provider/yql_kikimr_provider.cpp | 12 +++ ydb/core/kqp/ut/yql/kqp_yql_ut.cpp | 88 +++++++++++++++++++ ydb/library/uuid/uuid.cpp | 6 ++ ydb/library/uuid/uuid.h | 1 + 5 files changed, 113 insertions(+) diff --git a/ydb/core/kqp/opt/kqp_query_plan.cpp b/ydb/core/kqp/opt/kqp_query_plan.cpp index 86a1dfa0b1a6..c88a4e5ff50f 100644 --- a/ydb/core/kqp/opt/kqp_query_plan.cpp +++ b/ydb/core/kqp/opt/kqp_query_plan.cpp @@ -560,6 +560,12 @@ class TxPlanSerializer { } } + if (auto literal = key.Maybe()) { + TStringStream out; + NUuid::UuidBytesToString(literal.Cast().Literal().Value().Data(), out); + return out.Str(); + } + if (auto literal = key.Maybe()) { return literal.Cast().Literal().StringValue(); } diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index 8ba5403aa7ca..308e19695b7b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -605,6 +605,12 @@ void FillLiteralProtoImpl(const NNodes::TCoDataCtor& literal, TProto& proto) { protoValue.SetHi128(*reinterpret_cast(p + 8)); break; } + case EDataSlot::Uuid: { + const ui64* uuidData = reinterpret_cast(value.Data()); + protoValue.SetLow128(uuidData[0]); + protoValue.SetHi128(uuidData[1]); + break; + } default: YQL_ENSURE(false, "Unexpected type slot " << slot); @@ -738,6 +744,12 @@ void FillLiteralProto(const NNodes::TCoDataCtor& literal, Ydb::TypedValue& proto protoValue.set_high_128(*reinterpret_cast(p + 8)); break; } + case EDataSlot::Uuid: { + const ui64* uuidData = reinterpret_cast(value.Data()); + protoValue.set_low_128(uuidData[0]); + protoValue.set_high_128(uuidData[1]); + break; + } default: YQL_ENSURE(false, "Unexpected type slot " << slot); diff --git a/ydb/core/kqp/ut/yql/kqp_yql_ut.cpp b/ydb/core/kqp/ut/yql/kqp_yql_ut.cpp index 8ae6687ed686..9f1c68ac380e 100644 --- a/ydb/core/kqp/ut/yql/kqp_yql_ut.cpp +++ b/ydb/core/kqp/ut/yql/kqp_yql_ut.cpp @@ -740,6 +740,94 @@ Y_UNIT_TEST_SUITE(KqpYql) { } } + Y_UNIT_TEST(TestUuidPrimaryKeyPrefixSearch) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnablePreparedDdl(true); + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetAppConfig(appConfig) + .SetEnableUuidAsPrimaryKey(true) + .SetKqpSettings({setting}); + TKikimrRunner kikimr(serverSettings.SetWithSampleTables(false)); + + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + TVector testUuids = { + "5b99a330-04ef-4f1a-9b64-ba6d5f44eafe", + "afcbef30-9ac3-481a-aa6a-8d9b785dbb0a", + "b91cd23b-861c-4cc1-9119-801a4dac1cb9", + "65df9ecc-a97d-47b2-ae56-3c023da6ee8c", + }; + + { + const auto query = Q_(R"( + CREATE TABLE test( + key uuid NOT NULL, + val int, + PRIMARY KEY (key) + ); + )"); + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + { + int val = 0; + for (const auto& uuid : testUuids) { + const auto query = Sprintf("\ + INSERT INTO test (key, val)\n\ + VALUES (Uuid(\"%s\"), %u);\n\ + ", uuid.Data(), val++); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + } + { + int val = 0; + for (const auto& uuid : testUuids) { + const auto query = Sprintf("SELECT * FROM test WHERE key=Uuid(\"%s\");", uuid.Data()); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + TResultSetParser parser(result.GetResultSetParser(0)); + UNIT_ASSERT(parser.TryNextRow()); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().GetRef(), val++); + UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 1); + } + } + } + + Y_UNIT_TEST(TestUuidDefaultColumn) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnablePreparedDdl(true); + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetAppConfig(appConfig) + .SetEnableUuidAsPrimaryKey(true) + .SetKqpSettings({setting}); + TKikimrRunner kikimr(serverSettings.SetWithSampleTables(false)); + + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE test( + key int NOT NULL, + val uuid NOT NULL DEFAULT Uuid("65df9ecc-a97d-47b2-ae56-3c023da6ee8c"), + PRIMARY KEY (key) + ); + )"); + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + { + const auto query = "INSERT INTO test (key) VALUES (0);"; + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + } + Y_UNIT_TEST(UuidPrimaryKeyBulkUpsert) { auto settings = TKikimrSettings() .SetEnableUuidAsPrimaryKey(true) diff --git a/ydb/library/uuid/uuid.cpp b/ydb/library/uuid/uuid.cpp index aa12c7664b4c..ef1d2d44c00f 100644 --- a/ydb/library/uuid/uuid.cpp +++ b/ydb/library/uuid/uuid.cpp @@ -28,6 +28,12 @@ static void WriteHex(ui16 bytes, IOutputStream& out, bool reverseBytes = false) } } +void UuidBytesToString(TString in, IOutputStream& out) { + ui16 dw[8]; + std::memcpy(dw, in.Data(), sizeof(dw)); + NUuid::UuidToString(dw, out); +} + void UuidToString(ui16 dw[8], IOutputStream& out) { WriteHex(dw[1], out); WriteHex(dw[0], out); diff --git a/ydb/library/uuid/uuid.h b/ydb/library/uuid/uuid.h index abf7b53a7620..1d7ef1a4dfa9 100644 --- a/ydb/library/uuid/uuid.h +++ b/ydb/library/uuid/uuid.h @@ -13,6 +13,7 @@ namespace NUuid { static constexpr ui32 UUID_LEN = 16; +void UuidBytesToString(TString in, IOutputStream& out); void UuidToString(ui16 dw[8], IOutputStream& out); void UuidHalfsToByteString(ui64 low, ui64 hi, IOutputStream& out);