diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 85bb86ae179e..069b0a018b93 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -1805,6 +1805,7 @@ message TDescribeOptions { optional bool ReturnChannelsBinding = 8 [default = false]; optional bool ReturnRangeKey = 9 [default = true]; optional bool ReturnSetVal = 10 [default = false]; + optional bool ReturnIndexTableBoundaries = 11 [default = false]; } // Request to read scheme for a specific path diff --git a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp index 4af5a86d53ea..d6fca52aecaf 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp @@ -76,6 +76,7 @@ static NKikimrSchemeOp::TPathDescription GetTableDescription(TSchemeShard* ss, c opts.SetReturnPartitioningInfo(false); opts.SetReturnPartitionConfig(true); opts.SetReturnBoundaries(true); + opts.SetReturnIndexTableBoundaries(true); auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId, opts); auto record = desc->GetRecord(); diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index 2bbd78fea8c6..12e6a238570e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -72,6 +72,78 @@ static void FillTableStats(NKikimrSchemeOp::TPathDescription& pathDescription, c FillTableMetrics(pathDescription.MutableTabletMetrics(), stats); } +static void FillColumns( + const TTableInfo& tableInfo, + google::protobuf::RepeatedPtrField& out +) { + bool familyNamesBuilt = false; + THashMap familyNames; + + out.Reserve(tableInfo.Columns.size()); + for (const auto& col : tableInfo.Columns) { + const auto& cinfo = col.second; + if (cinfo.IsDropped()) + continue; + + auto* colDescr = out.Add(); + colDescr->SetName(cinfo.Name); + colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod)); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod); + colDescr->SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *colDescr->MutableTypeInfo() = *columnType.TypeInfo; + } + colDescr->SetId(cinfo.Id); + colDescr->SetNotNull(cinfo.NotNull); + + if (cinfo.Family != 0) { + colDescr->SetFamily(cinfo.Family); + + if (!familyNamesBuilt) { + for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) { + if (family.HasName() && family.HasId()) { + familyNames[family.GetId()] = family.GetName(); + } + } + familyNamesBuilt = true; + } + + auto it = familyNames.find(cinfo.Family); + if (it != familyNames.end() && !it->second.empty()) { + colDescr->SetFamilyName(it->second); + } + } + + colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress); + + switch (cinfo.DefaultKind) { + case ETableColumnDefaultKind::None: + break; + case ETableColumnDefaultKind::FromSequence: + colDescr->SetDefaultFromSequence(cinfo.DefaultValue); + break; + case ETableColumnDefaultKind::FromLiteral: + Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString( + cinfo.DefaultValue)); + break; + } + } +} + +static void FillKeyColumns( + const TTableInfo& tableInfo, + google::protobuf::RepeatedPtrField& names, + google::protobuf::RepeatedField& ids +) { + Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty()); + names.Reserve(tableInfo.KeyColumnIds.size()); + ids.Reserve(tableInfo.KeyColumnIds.size()); + for (ui32 keyColId : tableInfo.KeyColumnIds) { + *names.Add() = tableInfo.Columns.at(keyColId).Name; + *ids.Add() = keyColId; + } +} + void TPathDescriber::FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElement::TPtr pathEl, TPathElement::EPathSubType subType) { FillChildDescr(descr, pathEl); @@ -292,6 +364,7 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa bool returnBoundaries = false; bool returnRangeKey = true; bool returnSetVal = Params.GetOptions().GetReturnSetVal(); + bool returnIndexTableBoundaries = Params.GetOptions().GetReturnIndexTableBoundaries(); if (Params.HasOptions()) { returnConfig = Params.GetOptions().GetReturnPartitionConfig(); returnPartitioning = Params.GetOptions().GetReturnPartitioningInfo(); @@ -416,7 +489,9 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa switch (childPath->PathType) { case NKikimrSchemeOp::EPathTypeTableIndex: - Self->DescribeTableIndex(childPathId, childName, returnConfig, false, *entry->AddTableIndexes()); + Self->DescribeTableIndex( + childPathId, childName, returnConfig, returnIndexTableBoundaries, *entry->AddTableIndexes() + ); break; case NKikimrSchemeOp::EPathTypeCdcStream: Self->DescribeCdcStream(childPathId, childName, *entry->AddCdcStreams()); @@ -1175,67 +1250,10 @@ void TSchemeShard::DescribeTable( ) const { Y_UNUSED(typeRegistry); - THashMap familyNames; - bool familyNamesBuilt = false; entry->SetTableSchemaVersion(tableInfo->AlterVersion); - entry->MutableColumns()->Reserve(tableInfo->Columns.size()); - for (auto col : tableInfo->Columns) { - const auto& cinfo = col.second; - if (cinfo.IsDropped()) - continue; - - auto colDescr = entry->AddColumns(); - colDescr->SetName(cinfo.Name); - colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod)); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod); - colDescr->SetTypeId(columnType.TypeId); - if (columnType.TypeInfo) { - *colDescr->MutableTypeInfo() = *columnType.TypeInfo; - } - colDescr->SetId(cinfo.Id); - colDescr->SetNotNull(cinfo.NotNull); - - if (cinfo.Family != 0) { - colDescr->SetFamily(cinfo.Family); - - if (!familyNamesBuilt) { - for (const auto& family : tableInfo->PartitionConfig().GetColumnFamilies()) { - if (family.HasName() && family.HasId()) { - familyNames[family.GetId()] = family.GetName(); - } - } - familyNamesBuilt = true; - } - - auto it = familyNames.find(cinfo.Family); - if (it != familyNames.end() && !it->second.empty()) { - colDescr->SetFamilyName(it->second); - } - } - - colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress); - - switch (cinfo.DefaultKind) { - case ETableColumnDefaultKind::None: - break; - case ETableColumnDefaultKind::FromSequence: - colDescr->SetDefaultFromSequence(cinfo.DefaultValue); - break; - case ETableColumnDefaultKind::FromLiteral: - Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString( - cinfo.DefaultValue)); - break; - } - } - Y_ABORT_UNLESS(!tableInfo->KeyColumnIds.empty()); - - entry->MutableKeyColumnNames()->Reserve(tableInfo->KeyColumnIds.size()); - entry->MutableKeyColumnIds()->Reserve(tableInfo->KeyColumnIds.size()); - for (ui32 keyColId : tableInfo->KeyColumnIds) { - entry->AddKeyColumnNames(tableInfo->Columns[keyColId].Name); - entry->AddKeyColumnIds(keyColId); - } + FillColumns(*tableInfo, *entry->MutableColumns()); + FillKeyColumns(*tableInfo, *entry->MutableKeyColumnNames(), *entry->MutableKeyColumnIds()); if (fillConfig) { FillPartitionConfig(tableInfo->PartitionConfig(), *entry->MutablePartitionConfig()); @@ -1299,6 +1317,9 @@ void TSchemeShard::DescribeTableIndex(const TPathId& pathId, const TString& name FillPartitionConfig(tableInfo->PartitionConfig(), *tableDescription->MutablePartitionConfig()); } if (fillBoundaries) { + // column info is necessary for split boundary type conversion + FillColumns(*tableInfo, *tableDescription->MutableColumns()); + FillKeyColumns(*tableInfo, *tableDescription->MutableKeyColumnNames(), *tableDescription->MutableKeyColumnIds()); FillTableBoundaries(tableDescription->MutableSplitBoundary(), tableInfo); } } diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 2744041ea50d..f8b5b51c3321 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -839,8 +839,8 @@ void FillGlobalIndexSettings(Ydb::Table::GlobalIndexSettings& settings, NKikimrMiniKQL::TType splitKeyType; Ydb::Table::DescribeTableResult unused; FillColumnDescription(unused, splitKeyType, indexImplTableDescription); - FillTableBoundaryImpl( - *settings.mutable_partition_at_keys(), + FillTableBoundaryImpl( + settings, indexImplTableDescription, splitKeyType ); diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 656f0b087e00..917a11bbb1d4 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -485,6 +485,10 @@ class TTableDescription::TImpl { Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns)); } + void AddSecondaryIndex(const TIndexDescription& indexDescription) { + Indexes_.emplace_back(indexDescription); + } + void AddChangefeed(const TString& name, EChangefeedMode mode, EChangefeedFormat format) { Changefeeds_.emplace_back(name, mode, format); } @@ -757,6 +761,10 @@ void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType t Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns); } +void TTableDescription::AddSecondaryIndex(const TIndexDescription& indexDescription) { + Impl_->AddSecondaryIndex(indexDescription); +} + void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector& indexColumns) { AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns); } @@ -1191,6 +1199,11 @@ TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const TString& primaryKeyColum return *this; } +TTableBuilder& TTableBuilder::AddSecondaryIndex(const TIndexDescription& indexDescription) { + TableDescription_.AddSecondaryIndex(indexDescription); + return *this; +} + TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector& indexColumns, const TVector& dataColumns) { TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns); return *this; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index ba54abb380d8..1827c930457e 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -183,10 +183,10 @@ struct TExplicitPartitions { using TSelf = TExplicitPartitions; FLUENT_SETTING_VECTOR(TValue, SplitPoints); - + template static TExplicitPartitions FromProto(const TProto& proto); - + void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const; }; @@ -609,6 +609,7 @@ class TTableDescription { // common void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector& indexColumns); void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector& indexColumns, const TVector& dataColumns); + void AddSecondaryIndex(const TIndexDescription& indexDescription); // sync void AddSyncSecondaryIndex(const TString& indexName, const TVector& indexColumns); void AddSyncSecondaryIndex(const TString& indexName, const TVector& indexColumns, const TVector& dataColumns); @@ -820,6 +821,7 @@ class TTableBuilder { TTableBuilder& AddSerialColumn(const TString& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const TString& family = TString()); // common + TTableBuilder& AddSecondaryIndex(const TIndexDescription& indexDescription); TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector& indexColumns, const TVector& dataColumns); TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector& indexColumns); TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn); diff --git a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp index 8b489dbf5aba..f7e9371cf9c3 100644 --- a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp +++ b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp @@ -249,6 +249,68 @@ void TestTableSplitBoundariesArePreserved( UNIT_ASSERT_EQUAL(restoredTableDescription.GetKeyRanges(), originalKeyRanges); } +void TestIndexTableSplitBoundariesArePreserved( + const char* table, const char* index, ui64 indexPartitions, TSession& session, + TBackupFunction&& backup, TRestoreFunction&& restore +) { + const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable"); + + { + TExplicitPartitions indexPartitionBoundaries; + for (ui32 boundary : {1, 2, 4, 8, 16, 32, 64, 128, 256}) { + indexPartitionBoundaries.AppendSplitPoints( + // split boundary is technically always a tuple + TValueBuilder().BeginTuple().AddElement().OptionalUint32(boundary).EndTuple().Build() + ); + } + // By default indexImplTable has auto partitioning by size enabled, + // so you must set min partition count for partitions to not merge immediately after indexImplTable is built. + TPartitioningSettingsBuilder partitioningSettingsBuilder; + partitioningSettingsBuilder + .SetMinPartitionsCount(indexPartitions) + .SetMaxPartitionsCount(indexPartitions); + + const auto indexSettings = TGlobalIndexSettings{ + .PartitioningSettings = partitioningSettingsBuilder.Build(), + .Partitions = std::move(indexPartitionBoundaries) + }; + + auto tableBuilder = TTableBuilder() + .AddNullableColumn("Key", EPrimitiveType::Uint32) + .AddNullableColumn("Value", EPrimitiveType::Uint32) + .SetPrimaryKeyColumn("Key") + .AddSecondaryIndex(TIndexDescription("byValue", EIndexType::GlobalSync, { "Value" }, {}, { indexSettings })); + + const auto result = session.CreateTable(table, tableBuilder.Build()).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + const auto describeSettings = TDescribeTableSettings() + .WithTableStatistics(true) + .WithKeyShardBoundary(true); + const auto originalIndexTableDescription = GetTableDescription( + session, indexTablePath, describeSettings + ); + UNIT_ASSERT_VALUES_EQUAL(originalIndexTableDescription.GetPartitionsCount(), indexPartitions); + const auto& originalKeyRanges = originalIndexTableDescription.GetKeyRanges(); + UNIT_ASSERT_VALUES_EQUAL(originalKeyRanges.size(), indexPartitions); + + backup(table); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + + restore(table); + const auto restoredIndexTableDescription = GetTableDescription( + session, indexTablePath, describeSettings + ); + UNIT_ASSERT_VALUES_EQUAL(restoredIndexTableDescription.GetPartitionsCount(), indexPartitions); + const auto& restoredKeyRanges = restoredIndexTableDescription.GetKeyRanges(); + UNIT_ASSERT_VALUES_EQUAL(restoredKeyRanges.size(), indexPartitions); + UNIT_ASSERT_EQUAL(restoredKeyRanges, originalKeyRanges); +} + } Y_UNIT_TEST_SUITE(BackupRestore) { @@ -354,6 +416,7 @@ Y_UNIT_TEST_SUITE(BackupRestore) { ); } + // TO DO: test index impl table split boundaries restoration from a backup } Y_UNIT_TEST_SUITE(BackupRestoreS3) { @@ -544,4 +607,20 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) { ); } + Y_UNIT_TEST(RestoreIndexTableSplitBoundaries) { + TS3TestEnv testEnv; + constexpr const char* table = "/Root/table"; + constexpr const char* index = "byValue"; + constexpr ui64 indexPartitions = 10; + + TestIndexTableSplitBoundariesArePreserved( + table, + index, + indexPartitions, + testEnv.GetSession(), + CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), + CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) + ); + } + }