diff --git a/ydb/core/base/table_index.cpp b/ydb/core/base/table_index.cpp index 5c44338f4928..5198114c05c5 100644 --- a/ydb/core/base/table_index.cpp +++ b/ydb/core/base/table_index.cpp @@ -143,6 +143,14 @@ bool IsCompatibleIndex(NKikimrSchemeOp::EIndexType indexType, const TTableColumn return true; } +TVector GetImplTables(NKikimrSchemeOp::EIndexType indexType) { + if (indexType == NKikimrSchemeOp::EIndexType::EIndexTypeGlobalVectorKmeansTree) { + return { NTableVectorKmeansTreeIndex::LevelTable, NTableVectorKmeansTreeIndex::PostingTable }; + } else { + return { ImplTable }; + } +} + bool IsImplTable(std::string_view tableName) { return Contains(ImplTables, tableName); } diff --git a/ydb/core/base/table_index.h b/ydb/core/base/table_index.h index ce4f0a490edf..13caab312ec2 100644 --- a/ydb/core/base/table_index.h +++ b/ydb/core/base/table_index.h @@ -23,6 +23,8 @@ inline constexpr const char* ImplTable = "indexImplTable"; bool IsCompatibleIndex(NKikimrSchemeOp::EIndexType type, const TTableColumns& table, const TIndexColumns& index, TString& explain); TTableColumns CalcTableImplDescription(NKikimrSchemeOp::EIndexType type, const TTableColumns& table, const TIndexColumns& index); + +TVector GetImplTables(NKikimrSchemeOp::EIndexType indexType); bool IsImplTable(std::string_view tableName); bool IsTmpImplTable(std::string_view tableName); diff --git a/ydb/core/grpc_services/rpc_describe_table.cpp b/ydb/core/grpc_services/rpc_describe_table.cpp index 7847a7838795..06e7b5ac3754 100644 --- a/ydb/core/grpc_services/rpc_describe_table.cpp +++ b/ydb/core/grpc_services/rpc_describe_table.cpp @@ -28,8 +28,9 @@ class TDescribeTableRPC : public TRpcSchemeRequestActorAllowPrivateTableDescribeForTest) { return true; } - - if (path.EndsWith(TStringBuilder() << "/" << NTableIndex::ImplTable)) { + + auto pathElements = ::NKikimr::SplitPath(path); + if (pathElements.size() != 0 && NTableIndex::IsImplTable(pathElements.back())) { return true; } diff --git a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp index 90f870e5dd8c..def608c637a7 100644 --- a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp +++ b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp @@ -616,38 +616,39 @@ NThreading::TFuture TKqpTableMetadataLoader::LoadIndexMeta for (size_t i = 0; i < indexesCount; i++) { const auto& index = tableMetadata->Indexes[i]; - auto indexTablePath = NSchemeHelpers::CreateIndexTablePath(tableName, index.Name); - - if (!index.SchemaVersion) { - LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_GATEWAY, "Load index metadata without schema version check index: " << index.Name); - children.push_back( - LoadTableMetadata(cluster, indexTablePath, - TLoadTableMetadataSettings().WithPrivateTables(true), database, userToken) - .Apply([i, tableMetadata](const TFuture& result) { - auto value = result.GetValue(); - UpdateMetadataIfSuccess(tableMetadata, i, value); - return static_cast(value); - }) - ); + const auto indexTablePaths = NSchemeHelpers::CreateIndexTablePath(tableName, index.Type, index.Name); + for (const auto& indexTablePath : indexTablePaths) { + if (!index.SchemaVersion) { + LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_GATEWAY, "Load index metadata without schema version check index: " << index.Name); + children.push_back( + LoadTableMetadata(cluster, indexTablePath, + TLoadTableMetadataSettings().WithPrivateTables(true), database, userToken) + .Apply([i, tableMetadata](const TFuture& result) { + auto value = result.GetValue(); + UpdateMetadataIfSuccess(tableMetadata, i, value); + return static_cast(value); + }) + ); - } else { - LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_GATEWAY, "Load index metadata with schema version check" - << "index: " << index.Name - << "pathId: " << index.LocalPathId - << "ownerId: " << index.PathOwnerId - << "schemaVersion: " << index.SchemaVersion - << "tableOwnerId: " << tableOwnerId); - auto ownerId = index.PathOwnerId ? index.PathOwnerId : tableOwnerId; //for compat with 20-2 - children.push_back( - LoadIndexMetadataByPathId(cluster, - NKikimr::TIndexId(ownerId, index.LocalPathId, index.SchemaVersion), indexTablePath, database, userToken) - .Apply([i, tableMetadata](const TFuture& result) { - auto value = result.GetValue(); - UpdateMetadataIfSuccess(tableMetadata, i, value); - return static_cast(value); - }) - ); + } else { + LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_GATEWAY, "Load index metadata with schema version check" + << "index: " << index.Name + << "pathId: " << index.LocalPathId + << "ownerId: " << index.PathOwnerId + << "schemaVersion: " << index.SchemaVersion + << "tableOwnerId: " << tableOwnerId); + auto ownerId = index.PathOwnerId ? index.PathOwnerId : tableOwnerId; //for compat with 20-2 + children.push_back( + LoadIndexMetadataByPathId(cluster, + NKikimr::TIndexId(ownerId, index.LocalPathId, index.SchemaVersion), indexTablePath, database, userToken) + .Apply([i, tableMetadata](const TFuture& result) { + auto value = result.GetValue(); + UpdateMetadataIfSuccess(tableMetadata, i, value); + return static_cast(value); + }) + ); + } } } diff --git a/ydb/core/kqp/gateway/utils/scheme_helpers.cpp b/ydb/core/kqp/gateway/utils/scheme_helpers.cpp index 403d3f539ed1..eae21c0b0d43 100644 --- a/ydb/core/kqp/gateway/utils/scheme_helpers.cpp +++ b/ydb/core/kqp/gateway/utils/scheme_helpers.cpp @@ -46,8 +46,14 @@ bool SplitTablePath(const TString& tableName, const TString& database, std::pair } } -TString CreateIndexTablePath(const TString& tableName, const TString& indexName) { - return tableName + "/" + indexName + "/" + NTableIndex::ImplTable; +TVector CreateIndexTablePath(const TString& tableName, NYql::TIndexDescription::EType indexType, const TString& indexName) { + auto implTables = NTableIndex::GetImplTables(NYql::TIndexDescription::ConvertIndexType(indexType)); + TVector paths; + paths.reserve(implTables.size()); + for (const auto& implTable : implTables) { + paths.emplace_back(TStringBuilder() << tableName << "/" << indexName << "/" << implTable); + } + return paths; } bool SetDatabaseForLoginOperation(TString& result, bool getDomainLoginOnly, TMaybe domainName, diff --git a/ydb/core/kqp/gateway/utils/scheme_helpers.h b/ydb/core/kqp/gateway/utils/scheme_helpers.h index 21fe87528746..3566c0ba517b 100644 --- a/ydb/core/kqp/gateway/utils/scheme_helpers.h +++ b/ydb/core/kqp/gateway/utils/scheme_helpers.h @@ -22,7 +22,7 @@ bool TrySplitTablePath(const TString& path, std::pair& result, bool SplitTablePath(const TString& tableName, const TString& database, std::pair& pathPair, TString& error, bool createDir); -TString CreateIndexTablePath(const TString& tableName, const TString& indexName); +TVector CreateIndexTablePath(const TString& tableName, NYql::TIndexDescription::EType indexType, const TString& indexName); bool SetDatabaseForLoginOperation(TString& result, bool getDomainLoginOnly, TMaybe domainName, const TString& database); diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 58c2e24dbe77..28b226708874 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -1682,9 +1682,25 @@ class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformer().StringValue(); - auto indexTablePath = NKikimr::NKqp::NSchemeHelpers::CreateIndexTablePath(table.Metadata->Name, indexName); - alterTableRequest.set_path(std::move(indexTablePath)); + const auto indexName = indexSetting.Value().Cast().StringValue(); + const auto indexIter = std::find_if(table.Metadata->Indexes.begin(), table.Metadata->Indexes.end(), [&indexName] (const auto& index) { + return index.Name == indexName; + }); + if (indexIter == table.Metadata->Indexes.end()) { + ctx.AddError( + YqlIssue(ctx.GetPosition(indexSetting.Name().Pos()), + TIssuesIds::KIKIMR_SCHEME_ERROR, + TStringBuilder() << "Unknown index name: " << indexName)); + return SyncError(); + } + auto indexTablePaths = NKikimr::NKqp::NSchemeHelpers::CreateIndexTablePath(table.Metadata->Name, indexIter->Type, indexName); + if (indexTablePaths.size() != 1) { + ctx.AddError( + TIssue(ctx.GetPosition(indexSetting.Name().Pos()), + TStringBuilder() << "Only index with one impl table is supported")); + return SyncError(); + } + alterTableRequest.set_path(std::move(indexTablePaths[0])); } else if (settingName == "tableSettings") { auto tableSettings = indexSetting.Value().Cast(); for (const auto& tableSetting : tableSettings) { diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp index e3a814289926..59a4af7df35b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp @@ -130,13 +130,16 @@ struct TKiExploreTxResults { auto view = key.GetView(); if (view && view->Name) { const auto& indexName = view->Name; - const auto indexTablePath = NKikimr::NKqp::NSchemeHelpers::CreateIndexTablePath(tableMeta->Name, indexName); auto indexIt = std::find_if(tableMeta->Indexes.begin(), tableMeta->Indexes.end(), [&indexName](const auto& index){ return index.Name == indexName; }); YQL_ENSURE(indexIt != tableMeta->Indexes.end(), "Index not found"); + const auto indexTablePaths = NKikimr::NKqp::NSchemeHelpers::CreateIndexTablePath(tableMeta->Name, indexIt->Type, indexName); + YQL_ENSURE(indexTablePaths.size() == 1, "Only index with one impl table is supported"); + const auto indexTablePath = indexTablePaths[0]; + THashSet indexColumns; indexColumns.reserve(indexIt->KeyColumns.size() + indexIt->DataColumns.size()); for (const auto& keyColumn : indexIt->KeyColumns) { @@ -180,7 +183,9 @@ struct TKiExploreTxResults { continue; } - const auto indexTable = NKikimr::NKqp::NSchemeHelpers::CreateIndexTablePath(tableMeta->Name, index.Name); + const auto indexTables = NKikimr::NKqp::NSchemeHelpers::CreateIndexTablePath(tableMeta->Name, index.Type, index.Name); + YQL_ENSURE(indexTables.size() == 1, "Only index with one impl table is supported"); + const auto indexTable = indexTables[0]; ops[tableMeta->Name] |= TPrimitiveYdbOperation::Read; ops[indexTable] = TPrimitiveYdbOperation::Write; @@ -202,7 +207,10 @@ struct TKiExploreTxResults { continue; } - const auto indexTable = NKikimr::NKqp::NSchemeHelpers::CreateIndexTablePath(tableMeta->Name, index.Name); + const auto indexTables = NKikimr::NKqp::NSchemeHelpers::CreateIndexTablePath(tableMeta->Name, index.Type, index.Name); + YQL_ENSURE(indexTables.size() == 1, "Only index with one impl table is supported"); + const auto indexTable = indexTables[0]; + for (const auto& column : index.KeyColumns) { if (updateColumns.contains(column)) { // delete old index values and upsert rows into index table diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp index 1f598a272004..dc78070a64c8 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp +++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp @@ -502,6 +502,7 @@ void TKikimrRunner::Initialize(const TKikimrSettings& settings) { SetupLogLevelFromTestParam(NKikimrServices::KQP_TASKS_RUNNER); SetupLogLevelFromTestParam(NKikimrServices::KQP_EXECUTER); SetupLogLevelFromTestParam(NKikimrServices::TX_PROXY_SCHEME_CACHE); + SetupLogLevelFromTestParam(NKikimrServices::TX_PROXY); SetupLogLevelFromTestParam(NKikimrServices::SCHEME_BOARD_REPLICA); SetupLogLevelFromTestParam(NKikimrServices::KQP_WORKER); SetupLogLevelFromTestParam(NKikimrServices::KQP_SESSION); diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 4aefc8142460..debb858c4d98 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -2472,6 +2472,59 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } + Y_UNIT_TEST(AlterTableAlterVectorIndex) { + NKikimrConfig::TFeatureFlags featureFlags; + featureFlags.SetEnableVectorIndex(true); + auto settings = TKikimrSettings().SetFeatureFlags(featureFlags); + TKikimrRunner kikimr(settings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + { + TString create_index_query = R"( + --!syntax_v1 + CREATE TABLE `/Root/TestTable` ( + Key Uint64, + Embedding String, + PRIMARY KEY (Key), + INDEX vector_idx + GLOBAL USING vector_kmeans_tree + ON (Embedding) + WITH (similarity=cosine, vector_type=bit, vector_dimension=1) + ); + )"; + auto result = session.ExecuteSchemeQuery(create_index_query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto describe = session.DescribeTable("/Root/TestTable/vector_idx/indexImplPostingTable").GetValueSync(); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToString()); + auto indexDesc = describe.GetTableDescription(); + constexpr int defaultPartitionSizeMb = 2048; + UNIT_ASSERT_VALUES_EQUAL(indexDesc.GetPartitioningSettings().GetPartitionSizeMb(), defaultPartitionSizeMb); + } + { + auto result = session.ExecuteSchemeQuery(R"( + ALTER TABLE `/Root/TestTable` ALTER INDEX vector_idx SET AUTO_PARTITIONING_MIN_PARTITIONS_COUNT 1; + )").ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Only index with one impl table is supported" ); + } + } + + Y_UNIT_TEST(AlterTableAlterMissedIndex) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + { + auto result = session.ExecuteSchemeQuery(R"( + ALTER TABLE `/Root/SecondaryKeys` ALTER INDEX WrongIndexName SET AUTO_PARTITIONING_MIN_PARTITIONS_COUNT 1; + )").ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Unknown index name: WrongIndexName"); + } + } + Y_UNIT_TEST(AlterIndexImplTable) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); @@ -2695,6 +2748,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { WITH (similarity=inner_product, vector_type=float, vector_dimension=1024) ); )"; + auto result = session.ExecuteSchemeQuery(create_index_query).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -2714,7 +2768,14 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL(indexDesc.GetVectorIndexSettings()->VectorType, NYdb::NTable::TVectorIndexSettings::EVectorType::Float); UNIT_ASSERT_VALUES_EQUAL(indexDesc.GetVectorIndexSettings()->VectorDimension, 1024); } - } + { + auto describeLevelTable = session.DescribeTable("/Root/TestTable/vector_idx/indexImplLevelTable").GetValueSync(); + UNIT_ASSERT_C(describeLevelTable.IsSuccess(), describeLevelTable.GetIssues().ToString()); + auto describePostingTable = session.DescribeTable("/Root/TestTable/vector_idx/indexImplPostingTable").GetValueSync(); + UNIT_ASSERT_C(describePostingTable.IsSuccess(), describePostingTable.GetIssues().ToString()); + } + } + Y_UNIT_TEST(CreateTableWithVectorIndexCovered) { NKikimrConfig::TFeatureFlags featureFlags;