From fbcefe2b710d54ca7049a78d5b968e2e206d9cd7 Mon Sep 17 00:00:00 2001 From: Daniil Demin Date: Tue, 17 Sep 2024 17:22:53 +0300 Subject: [PATCH 1/2] Capture TablePathPrefix (and other parts of the parser context) in CREATE VIEW (#8991) --- .../kqp/gateway/behaviour/view/manager.cpp | 19 ++--- ydb/core/kqp/gateway/behaviour/view/ya.make | 1 + ydb/core/kqp/gateway/kqp_metadata_loader.cpp | 2 +- ydb/core/kqp/host/kqp_gateway_proxy.cpp | 1 + ydb/core/kqp/host/kqp_host.cpp | 14 +++- ydb/core/kqp/host/kqp_translate.cpp | 7 +- ydb/core/kqp/host/kqp_translate.h | 3 +- ydb/core/kqp/provider/rewrite_io_utils.cpp | 16 +++-- ydb/core/kqp/provider/rewrite_io_utils.h | 7 +- ydb/core/kqp/provider/yql_kikimr_datasink.cpp | 6 +- .../kqp/provider/yql_kikimr_datasource.cpp | 7 +- ydb/core/kqp/provider/yql_kikimr_gateway.h | 2 + ydb/core/kqp/provider/yql_kikimr_provider.cpp | 33 +++++++++ ydb/core/kqp/provider/yql_kikimr_provider.h | 10 +++ ydb/core/kqp/ut/view/view_ut.cpp | 67 ++++++++++++++++-- ydb/core/protos/flat_scheme_op.proto | 2 + ydb/core/protos/ya.make | 1 + .../protos/yql_translation_settings.proto | 14 ++++ ydb/core/tx/schemeshard/schemeshard__init.cpp | 3 + .../schemeshard__operation_create_view.cpp | 5 +- ydb/core/tx/schemeshard/schemeshard_impl.cpp | 4 +- .../tx/schemeshard/schemeshard_info_types.h | 2 + .../schemeshard_path_describer.cpp | 1 + ydb/core/tx/schemeshard/schemeshard_schema.h | 4 +- ydb/library/yql/sql/sql.cpp | 14 ++-- ydb/library/yql/sql/sql.h | 3 +- ydb/library/yql/sql/v1/sql_query.cpp | 8 ++- ydb/library/yql/sql/v1/sql_translation.cpp | 70 ++++++++++++++----- ydb/services/metadata/manager/abstract.h | 2 + ydb/services/metadata/manager/ya.make | 1 + .../flat_schemeshard.schema | 8 ++- 31 files changed, 274 insertions(+), 63 deletions(-) create mode 100644 ydb/core/protos/yql_translation_settings.proto diff --git a/ydb/core/kqp/gateway/behaviour/view/manager.cpp b/ydb/core/kqp/gateway/behaviour/view/manager.cpp index 236125c0e85a..fa052618ff91 100644 --- a/ydb/core/kqp/gateway/behaviour/view/manager.cpp +++ b/ydb/core/kqp/gateway/behaviour/view/manager.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace NKikimr::NKqp { @@ -11,13 +12,14 @@ namespace { using TYqlConclusionStatus = TViewManager::TYqlConclusionStatus; using TInternalModificationContext = TViewManager::TInternalModificationContext; +using TExternalModificationContext = TViewManager::TExternalModificationContext; TString GetByKeyOrDefault(const NYql::TCreateObjectSettings& container, const TString& key) { const auto value = container.GetFeaturesExtractor().Extract(key); return value ? *value : TString{}; } -TYqlConclusionStatus CheckFeatureFlag(TInternalModificationContext& context) { +TYqlConclusionStatus CheckFeatureFlag(const TInternalModificationContext& context) { auto* const actorSystem = context.GetExternalData().GetActorSystem(); if (!actorSystem) { ythrow yexception() << "This place needs an actor system. Please contact internal support"; @@ -48,15 +50,16 @@ std::pair SplitPathByObjectId(const TString& objectId) { void FillCreateViewProposal(NKikimrSchemeOp::TModifyScheme& modifyScheme, const NYql::TCreateObjectSettings& settings, - const TString& database) { + const TExternalModificationContext& context) { - const auto pathPair = SplitPathByDb(settings.GetObjectId(), database); + const auto pathPair = SplitPathByDb(settings.GetObjectId(), context.GetDatabase()); modifyScheme.SetWorkingDir(pathPair.first); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateView); auto& viewDesc = *modifyScheme.MutableCreateView(); viewDesc.SetName(pathPair.second); viewDesc.SetQueryText(GetByKeyOrDefault(settings, "query_text")); + NSQLTranslation::Serialize(context.GetTranslationSettings(), *viewDesc.MutableCapturedContext()); if (!settings.GetFeaturesExtractor().IsFinished()) { ythrow TBadArgumentException() << "Unknown property: " << settings.GetFeaturesExtractor().GetRemainedParamsString(); @@ -92,20 +95,20 @@ NThreading::TFuture SendSchemeRequest(TEvTxUserProxy::TEvP } NThreading::TFuture CreateView(const NYql::TCreateObjectSettings& settings, - TInternalModificationContext& context) { + const TInternalModificationContext& context) { auto proposal = MakeHolder(); proposal->Record.SetDatabaseName(context.GetExternalData().GetDatabase()); if (context.GetExternalData().GetUserToken()) { proposal->Record.SetUserToken(context.GetExternalData().GetUserToken()->GetSerializedToken()); } auto& schemeTx = *proposal->Record.MutableTransaction()->MutableModifyScheme(); - FillCreateViewProposal(schemeTx, settings, context.GetExternalData().GetDatabase()); + FillCreateViewProposal(schemeTx, settings, context.GetExternalData()); return SendSchemeRequest(proposal.Release(), context.GetExternalData().GetActorSystem(), true); } NThreading::TFuture DropView(const NYql::TDropObjectSettings& settings, - TInternalModificationContext& context) { + const TInternalModificationContext& context) { auto proposal = MakeHolder(); proposal->Record.SetDatabaseName(context.GetExternalData().GetDatabase()); if (context.GetExternalData().GetUserToken()) { @@ -119,8 +122,8 @@ NThreading::TFuture DropView(const NYql::TDropObjectSettin void PrepareCreateView(NKqpProto::TKqpSchemeOperation& schemeOperation, const NYql::TObjectSettingsImpl& settings, - TInternalModificationContext& context) { - FillCreateViewProposal(*schemeOperation.MutableCreateView(), settings, context.GetExternalData().GetDatabase()); + const TInternalModificationContext& context) { + FillCreateViewProposal(*schemeOperation.MutableCreateView(), settings, context.GetExternalData()); } void PrepareDropView(NKqpProto::TKqpSchemeOperation& schemeOperation, diff --git a/ydb/core/kqp/gateway/behaviour/view/ya.make b/ydb/core/kqp/gateway/behaviour/view/ya.make index 6cb342036bda..7d57b8ceaab8 100644 --- a/ydb/core/kqp/gateway/behaviour/view/ya.make +++ b/ydb/core/kqp/gateway/behaviour/view/ya.make @@ -8,6 +8,7 @@ SRCS( PEERDIR( ydb/core/base ydb/core/kqp/gateway/actors + ydb/core/kqp/provider ydb/core/tx/tx_proxy ydb/services/metadata/abstract ydb/services/metadata/manager diff --git a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp index 23ea354852a9..db99faa189bd 100644 --- a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp +++ b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp @@ -303,7 +303,7 @@ TTableMetadataResult GetViewMetadataResult( metadata->SchemaVersion = description.GetVersion(); metadata->Kind = NYql::EKikimrTableKind::View; metadata->Attributes = schemeEntry.Attributes; - metadata->ViewPersistedData = {description.GetQueryText()}; + metadata->ViewPersistedData = {description.GetQueryText(), description.GetCapturedContext()}; return builtResult; } diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp index 612055b3ad17..78daaa4004c1 100644 --- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp +++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp @@ -1266,6 +1266,7 @@ class TKqpGatewayProxy : public IKikimrGateway { if (SessionCtx->GetUserToken()) { context.SetUserToken(*SessionCtx->GetUserToken()); } + context.SetTranslationSettings(SessionCtx->Query().TranslationSettings); auto& phyTx = phyTxRemover.Capture(SessionCtx->Query().PreparingQuery->MutablePhysicalQuery()); phyTx.SetType(NKqpProto::TKqpPhyTx::TYPE_SCHEME); diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index 3e98f7db8aa3..2e6b9141ceb8 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -1228,7 +1228,19 @@ class TKqpHost : public IKqpHost { .SetQueryParameters(query.ParameterTypes) .SetApplicationName(ApplicationName) .SetIsEnablePgSyntax(SessionCtx->Config().FeatureFlags.GetEnablePgSyntax()); - auto astRes = ParseQuery(query.Text, isSql, sqlVersion, TypesCtx->DeprecatedSQL, ctx, settingsBuilder, result.KeepInCache, result.CommandTagName); + NSQLTranslation::TTranslationSettings effectiveSettings; + auto astRes = ParseQuery( + query.Text, + isSql, + sqlVersion, + TypesCtx->DeprecatedSQL, + ctx, + settingsBuilder, + result.KeepInCache, + result.CommandTagName, + &effectiveSettings + ); + SessionCtx->Query().TranslationSettings = std::move(effectiveSettings); if (astRes.ActualSyntaxType == NYql::ESyntaxType::Pg) { SessionCtx->Config().IndexAutoChooserMode = NKikimrConfig::TTableServiceConfig_EIndexAutoChooseMode::TTableServiceConfig_EIndexAutoChooseMode_MAX_USED_PREFIX; } diff --git a/ydb/core/kqp/host/kqp_translate.cpp b/ydb/core/kqp/host/kqp_translate.cpp index 8607f5d87d36..8b7d842f88ed 100644 --- a/ydb/core/kqp/host/kqp_translate.cpp +++ b/ydb/core/kqp/host/kqp_translate.cpp @@ -54,7 +54,7 @@ NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type) { YQL_ENSURE(false, "Unexpected query type: " << type); } } - + NSQLTranslation::TTranslationSettings TKqpTranslationSettingsBuilder::Build(NYql::TExprContext& ctx) { NSQLTranslation::TTranslationSettings settings; settings.PgParser = UsePgParser && *UsePgParser; @@ -154,13 +154,14 @@ NSQLTranslation::TTranslationSettings TKqpTranslationSettingsBuilder::Build(NYql } NYql::TAstParseResult ParseQuery(const TString& queryText, bool isSql, TMaybe& sqlVersion, bool& deprecatedSQL, - NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe& commandTagName) { + NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe& commandTagName, + NSQLTranslation::TTranslationSettings* effectiveSettings) { NYql::TAstParseResult astRes; settingsBuilder.SetSqlVersion(sqlVersion); if (isSql) { auto settings = settingsBuilder.Build(ctx); NYql::TStmtParseInfo stmtParseInfo; - auto ast = NSQLTranslation::SqlToYql(queryText, settings, nullptr, &stmtParseInfo); + auto ast = NSQLTranslation::SqlToYql(queryText, settings, nullptr, &stmtParseInfo, effectiveSettings); deprecatedSQL = (ast.ActualSyntaxType == NYql::ESyntaxType::YQLv0); sqlVersion = ast.ActualSyntaxType == NYql::ESyntaxType::YQLv1 ? 1 : 0; keepInCache = stmtParseInfo.KeepInCache; diff --git a/ydb/core/kqp/host/kqp_translate.h b/ydb/core/kqp/host/kqp_translate.h index 033a79e166e3..d71c01d4ce2b 100644 --- a/ydb/core/kqp/host/kqp_translate.h +++ b/ydb/core/kqp/host/kqp_translate.h @@ -91,7 +91,8 @@ NSQLTranslation::EBindingsMode RemapBindingsMode(NKikimrConfig::TTableServiceCon NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type); NYql::TAstParseResult ParseQuery(const TString& queryText, bool isSql, TMaybe& sqlVersion, bool& deprecatedSQL, - NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe& commandTagName); + NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe& commandTagName, + NSQLTranslation::TTranslationSettings* effectiveSettings = nullptr); TVector ParseStatements(const TString& queryText, const TMaybe& syntax, bool isSql, TKqpTranslationSettingsBuilder& settingsBuilder, bool perStatementExecution); diff --git a/ydb/core/kqp/provider/rewrite_io_utils.cpp b/ydb/core/kqp/provider/rewrite_io_utils.cpp index 9a2bd768904b..067ab4b583a9 100644 --- a/ydb/core/kqp/provider/rewrite_io_utils.cpp +++ b/ydb/core/kqp/provider/rewrite_io_utils.cpp @@ -1,6 +1,7 @@ #include "rewrite_io_utils.h" #include +#include #include #include #include @@ -16,16 +17,17 @@ using namespace NNodes; constexpr const char* QueryGraphNodeSignature = "SavedQueryGraph"; TExprNode::TPtr CompileViewQuery( - const TString& query, TExprContext& ctx, NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder, - IModuleResolver::TPtr moduleResolver + IModuleResolver::TPtr moduleResolver, + const TViewPersistedData& viewData ) { auto translationSettings = settingsBuilder.Build(ctx); translationSettings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW; + NSQLTranslation::Deserialize(viewData.CapturedContext, translationSettings); TAstParseResult queryAst; - queryAst = NSQLTranslation::SqlToYql(query, translationSettings); + queryAst = NSQLTranslation::SqlToYql(viewData.QueryText, translationSettings); ctx.IssueManager.AddIssues(queryAst.Issues); if (!queryAst.IsOk()) { @@ -116,9 +118,9 @@ TExprNode::TPtr FindTopLevelRead(const TExprNode::TPtr& queryGraph) { TExprNode::TPtr RewriteReadFromView( const TExprNode::TPtr& node, TExprContext& ctx, - const TString& query, NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder, - IModuleResolver::TPtr moduleResolver + IModuleResolver::TPtr moduleResolver, + const TViewPersistedData& viewData ) { YQL_PROFILE_FUNC(DEBUG); @@ -127,7 +129,7 @@ TExprNode::TPtr RewriteReadFromView( TExprNode::TPtr queryGraph = FindSavedQueryGraph(readNode.Ptr()); if (!queryGraph) { - queryGraph = CompileViewQuery(query, ctx, settingsBuilder, moduleResolver); + queryGraph = CompileViewQuery(ctx, settingsBuilder, moduleResolver, viewData); if (!queryGraph) { ctx.AddError(TIssue(ctx.GetPosition(readNode.Pos()), "The query stored in the view cannot be compiled.")); @@ -151,4 +153,4 @@ TExprNode::TPtr RewriteReadFromView( return Build(ctx, node->Pos()).Input(topLevelRead).Done().Ptr(); } -} \ No newline at end of file +} diff --git a/ydb/core/kqp/provider/rewrite_io_utils.h b/ydb/core/kqp/provider/rewrite_io_utils.h index 71beb4e56c22..dd3dff66dd06 100644 --- a/ydb/core/kqp/provider/rewrite_io_utils.h +++ b/ydb/core/kqp/provider/rewrite_io_utils.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include namespace NYql { @@ -10,9 +11,9 @@ TExprNode::TPtr FindTopLevelRead(const TExprNode::TPtr& queryGraph); TExprNode::TPtr RewriteReadFromView( const TExprNode::TPtr& node, TExprContext& ctx, - const TString& query, NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder, - IModuleResolver::TPtr moduleResolver + IModuleResolver::TPtr moduleResolver, + const TViewPersistedData& viewData ); -} \ No newline at end of file +} diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index ea314fe13fab..e418a96804d6 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -176,9 +176,9 @@ class TKiSinkIntentDeterminationTransformer: public TKiSinkVisitorTransformer { } TStatus HandleDropObject(TKiDropObject node, TExprContext& ctx) override { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() - << "DropObject is not yet implemented for intent determination transformer")); - return TStatus::Error; + Y_UNUSED(node); + Y_UNUSED(ctx); + return TStatus::Ok; } TStatus HandleCreateGroup(TKiCreateGroup node, TExprContext& ctx) override { diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp index 3a0ba86838f7..1c3a02efc504 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp @@ -771,16 +771,17 @@ class TKikimrDataSource : public TDataProviderBase { .Repeat(TExprStep::LoadTablesMetadata) .Repeat(TExprStep::RewriteIO); - const auto& query = tableDesc.Metadata->ViewPersistedData.QueryText; + const auto& viewData = tableDesc.Metadata->ViewPersistedData; + NKqp::TKqpTranslationSettingsBuilder settingsBuilder( SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), cluster, - query, + viewData.QueryText, SessionCtx->Config().BindingsMode, GUCSettings ); - return RewriteReadFromView(node, ctx, query, settingsBuilder, Types.Modules); + return RewriteReadFromView(node, ctx, settingsBuilder, Types.Modules, viewData); } } diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 724609282da7..062ab15a8e35 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -406,6 +407,7 @@ enum EMetaSerializationType : ui64 { struct TViewPersistedData { TString QueryText; + NYql::NProto::TTranslationSettings CapturedContext; }; struct TKikimrTableMetadata : public TThrRefBase { diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index ac310064077c..b71e0082277e 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -964,3 +964,36 @@ TCoNameValueTupleList TKiExecDataQuerySettings::BuildNode(TExprContext& ctx, TPo } } // namespace NYql + +namespace NSQLTranslation { + +void Serialize(const TTranslationSettings& settings, NYql::NProto::TTranslationSettings& serializedSettings) { + serializedSettings.SetPathPrefix(settings.PathPrefix); + serializedSettings.SetSyntaxVersion(settings.SyntaxVersion); + serializedSettings.SetAnsiLexer(settings.AnsiLexer); + serializedSettings.SetPgParser(settings.PgParser); + + auto* pragmas = serializedSettings.MutablePragmas(); + pragmas->Clear(); + pragmas->Add(settings.Flags.begin(), settings.Flags.end()); +} + +void Deserialize(const NYql::NProto::TTranslationSettings& serializedSettings, TTranslationSettings& settings) { + #define DeserializeSetting(settingName) \ + if (serializedSettings.Has##settingName()) { \ + settings.settingName = serializedSettings.Get##settingName(); \ + } + + DeserializeSetting(PathPrefix); + DeserializeSetting(SyntaxVersion); + DeserializeSetting(AnsiLexer); + DeserializeSetting(PgParser); + + #undef DeserializeSetting + + // overwrite existing pragmas + settings.Flags.clear(); + settings.Flags.insert(serializedSettings.GetPragmas().begin(), serializedSettings.GetPragmas().end()); +} + +} diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h index cdca12b6a156..3c1ad3edf62f 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider.h @@ -124,6 +124,8 @@ struct TKikimrQueryContext : TThrRefBase { // we do not want add extra life time for query context here std::shared_ptr RpcCtx; + NSQLTranslation::TTranslationSettings TranslationSettings; + void Reset() { PrepareOnly = false; SuppressDdlChecks = false; @@ -142,6 +144,7 @@ struct TKikimrQueryContext : TThrRefBase { RlPath.Clear(); RpcCtx.reset(); + TranslationSettings = NSQLTranslation::TTranslationSettings(); } }; @@ -578,3 +581,10 @@ TIntrusivePtr CreateKikimrDataSink( TIntrusivePtr queryExecutor); } // namespace NYql + +namespace NSQLTranslation { + +void Serialize(const TTranslationSettings& settings, NYql::NProto::TTranslationSettings& serializedSettings); +void Deserialize(const NYql::NProto::TTranslationSettings& serializedSettings, TTranslationSettings& settings); + +} diff --git a/ydb/core/kqp/ut/view/view_ut.cpp b/ydb/core/kqp/ut/view/view_ut.cpp index 056a16b7958f..05002dd05ed0 100644 --- a/ydb/core/kqp/ut/view/view_ut.cpp +++ b/ydb/core/kqp/ut/view/view_ut.cpp @@ -56,6 +56,20 @@ TString ReadWholeFile(const TString& path) { return file.ReadAll(); } +NQuery::TExecuteQueryResult ExecuteQuery(NQuery::TSession& session, const TString& query) { + const auto result = session.ExecuteQuery( + query, + NQuery::TTxControl::NoTx() + ).ExtractValueSync(); + + UNIT_ASSERT_C(result.IsSuccess(), + "Failed to execute the following query:\n" << query << '\n' + << "The issues:\n" << result.GetIssues().ToString() + ); + + return result; +} + void ExecuteDataDefinitionQuery(TSession& session, const TString& script) { const auto result = session.ExecuteSchemeQuery(script).ExtractValueSync(); UNIT_ASSERT_C(result.IsSuccess(), "Failed to execute the following DDL script:\n" @@ -110,16 +124,21 @@ void AssertFromCache(const TMaybe& stats, bool expectedValue) { UNIT_ASSERT_VALUES_EQUAL_C(*isFromCache, expectedValue, stats->ToString()); } -void CompareResults(const TDataQueryResult& first, const TDataQueryResult& second) { - const auto& firstResults = first.GetResultSets(); - const auto& secondResults = second.GetResultSets(); - +void CompareResults(const TVector& firstResults, const TVector& secondResults) { UNIT_ASSERT_VALUES_EQUAL(firstResults.size(), secondResults.size()); for (size_t i = 0; i < firstResults.size(); ++i) { CompareYson(FormatResultSetYson(firstResults[i]), FormatResultSetYson(secondResults[i])); } } +void CompareResults(const TDataQueryResult& first, const TDataQueryResult& second) { + CompareResults(first.GetResultSets(), second.GetResultSets()); +} + +void CompareResults(const NQuery::TExecuteQueryResult& first, const NQuery::TExecuteQueryResult& second) { + CompareResults(first.GetResultSets(), second.GetResultSets()); +} + void InitializeTablesAndSecondaryViews(TSession& session) { const auto inputFolder = ArcadiaFromCurrentLocation(__SOURCE_FILE__, "input"); ExecuteDataDefinitionQuery(session, ReadWholeFile(inputFolder + "/create_tables_and_secondary_views.sql")); @@ -408,6 +427,46 @@ Y_UNIT_TEST_SUITE(TSelectFromViewTest) { CompareResults(etalonResults, selectFromViewResults); } + Y_UNIT_TEST(OneTableUsingRelativeName) { + TKikimrRunner kikimr; + + auto& runtime = *kikimr.GetTestServer().GetRuntime(); + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NLog::PRI_DEBUG); + + EnableViewsFeatureFlag(kikimr); + auto session = kikimr.GetQueryClient().GetSession().ExtractValueSync().GetSession(); + + constexpr const char* viewName = "TheView"; + constexpr const char* testTable = "Test"; + const auto innerQuery = std::format(R"( + SELECT * FROM {} + )", + testTable + ); + + const TString creationQuery = std::format(R"( + CREATE VIEW {} WITH (security_invoker = true) AS {}; + )", + viewName, + innerQuery + ); + ExecuteQuery(session, creationQuery); + + const auto etalonResults = ExecuteQuery(session, std::format(R"( + SELECT * FROM ({}); + )", + innerQuery + ) + ); + const auto selectFromViewResults = ExecuteQuery(session, std::format(R"( + SELECT * FROM {}; + )", + viewName + ) + ); + CompareResults(etalonResults, selectFromViewResults); + } + Y_UNIT_TEST(DisabledFeatureFlag) { TKikimrRunner kikimr(TKikimrSettings().SetWithSampleTables(false)); auto session = kikimr.GetTableClient().CreateSession().GetValueSync().GetSession(); diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 1d385713a62a..0526afec824d 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -11,6 +11,7 @@ import "ydb/core/protos/filestore_config.proto"; import "ydb/core/protos/channel_purpose.proto"; import "ydb/core/protos/follower_group.proto"; import "ydb/core/protos/blob_depot_config.proto"; +import "ydb/core/protos/yql_translation_settings.proto"; import "ydb/public/api/protos/ydb_coordination.proto"; import "ydb/public/api/protos/ydb_export.proto"; import "ydb/public/api/protos/ydb_value.proto"; @@ -2113,6 +2114,7 @@ message TViewDescription { optional NKikimrProto.TPathID PathId = 2; optional uint64 Version = 3; optional string QueryText = 4; + optional NYql.NProto.TTranslationSettings CapturedContext = 5; } message TResourcePoolProperties { diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index aa55a2d8a046..039516a68131 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -144,6 +144,7 @@ SRCS( tx_sequenceshard.proto ydb_result_set_old.proto ydb_table_impl.proto + yql_translation_settings.proto ) GENERATE_ENUM_SERIALIZATION(blobstorage_pdisk_config.pb.h) diff --git a/ydb/core/protos/yql_translation_settings.proto b/ydb/core/protos/yql_translation_settings.proto new file mode 100644 index 000000000000..515900b14d29 --- /dev/null +++ b/ydb/core/protos/yql_translation_settings.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package NYql.NProto; + +option java_package = "ru.yandex.kikimr.proto"; + +message TTranslationSettings { + optional string PathPrefix = 1; + optional uint32 SyntaxVersion = 2; + optional bool AnsiLexer = 3; + optional bool PgParser = 4; + + repeated string Pragmas = 10; +} diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index f5f3cb1bbed3..5c847eb7a0ca 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -1905,6 +1905,9 @@ struct TSchemeShard::TTxInit : public TTransactionBase { auto& view = Self->Views[pathId] = new TViewInfo(); view->AlterVersion = rowset.GetValue(); view->QueryText = rowset.GetValue(); + Y_PROTOBUF_SUPPRESS_NODISCARD view->CapturedContext.ParseFromString( + rowset.GetValue() + ); Self->IncrementPathDbRefCount(pathId); if (!rowset.Next()) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_view.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_view.cpp index e4f6e69922ba..0e172572f778 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_view.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_view.cpp @@ -48,7 +48,7 @@ class TPropose: public TSubOperationState { Y_ABORT_UNLESS(txState->TxType == TTxState::TxCreateView); context.SS->TabletCounters->Simple()[COUNTER_VIEW_COUNT].Add(1); - + const auto pathId = txState->TargetPathId; auto path = TPath::Init(pathId, context.SS); @@ -68,6 +68,7 @@ TViewInfo::TPtr CreateView(const NKikimrSchemeOp::TViewDescription& desc) { TViewInfo::TPtr viewInfo = new TViewInfo; viewInfo->AlterVersion = 1; viewInfo->QueryText = desc.GetQueryText(); + viewInfo->CapturedContext = desc.GetCapturedContext(); return viewInfo; } @@ -109,7 +110,7 @@ class TCreateView: public TSubOperation { const auto& viewDescription = Transaction.GetCreateView(); const TString& name = viewDescription.GetName(); - + LOG_N("TCreateView Propose" << ", path: " << parentPathStr << "/" << name << ", opId: " << OperationId diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 594cd794d368..4242d06478e7 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -2955,7 +2955,9 @@ void TSchemeShard::PersistView(NIceDb::TNiceDb &db, TPathId pathId) { db.Table().Key(pathId.LocalPathId).Update( NIceDb::TUpdate{viewInfo->AlterVersion}, - NIceDb::TUpdate{viewInfo->QueryText}); + NIceDb::TUpdate{viewInfo->QueryText}, + NIceDb::TUpdate{viewInfo->CapturedContext.SerializeAsString()} + ); } void TSchemeShard::PersistRemoveView(NIceDb::TNiceDb& db, TPathId pathId) { diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 890807a1e899..cabba3c817e7 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -3276,6 +3277,7 @@ struct TViewInfo : TSimpleRefCount { ui64 AlterVersion = 0; TString QueryText; + NYql::NProto::TTranslationSettings CapturedContext; }; struct TResourcePoolInfo : TSimpleRefCount { diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index 34ecf178b3bf..69cdc88478a3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -959,6 +959,7 @@ void TPathDescriber::DescribeView(const TActorContext&, TPathId pathId, TPathEle PathIdFromPathId(pathId, entry->MutablePathId()); entry->SetVersion(viewInfo->AlterVersion); entry->SetQueryText(viewInfo->QueryText); + *entry->MutableCapturedContext() = viewInfo->CapturedContext; } void TPathDescriber::DescribeResourcePool(TPathId pathId, TPathElement::TPtr pathEl) { diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index d36c6d4f6532..14ecc6cc8e44 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1783,9 +1783,11 @@ struct Schema : NIceDb::Schema { struct PathId: Column<1, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; struct AlterVersion: Column<2, NScheme::NTypeIds::Uint64> {}; struct QueryText: Column<3, NScheme::NTypeIds::String> {}; + // CapturedContext is a serialized NYql::NProto::TTranslationSettings. + struct CapturedContext: Column<4, NScheme::NTypeIds::String> {}; using TKey = TableKey; - using TColumns = TableColumns; + using TColumns = TableColumns; }; struct BackgroundSessions: Table<109> { diff --git a/ydb/library/yql/sql/sql.cpp b/ydb/library/yql/sql/sql.cpp index c981cd690a7e..2c8702d38035 100644 --- a/ydb/library/yql/sql/sql.cpp +++ b/ydb/library/yql/sql/sql.cpp @@ -15,18 +15,22 @@ namespace NSQLTranslation { NYql::TAstParseResult SqlToYql(const TString& query, const TTranslationSettings& settings, - NYql::TWarningRules* warningRules, NYql::TStmtParseInfo* stmtParseInfo) + NYql::TWarningRules* warningRules, NYql::TStmtParseInfo* stmtParseInfo, TTranslationSettings* effectiveSettings) { NYql::TAstParseResult result; TTranslationSettings parsedSettings(settings); - google::protobuf::Arena arena; - if (!parsedSettings.Arena) { - parsedSettings.Arena = &arena; - } if (!ParseTranslationSettings(query, parsedSettings, result.Issues)) { return result; } + if (effectiveSettings) { + *effectiveSettings = parsedSettings; + } + + google::protobuf::Arena arena; + if (!parsedSettings.Arena) { + parsedSettings.Arena = &arena; + } if (!parsedSettings.DeclaredNamedExprs.empty() && !parsedSettings.PgParser && parsedSettings.SyntaxVersion != 1) { result.Issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, diff --git a/ydb/library/yql/sql/sql.h b/ydb/library/yql/sql/sql.h index f00252a5faec..eecdde9bae36 100644 --- a/ydb/library/yql/sql/sql.h +++ b/ydb/library/yql/sql/sql.h @@ -16,7 +16,8 @@ namespace NSQLTranslation { NYql::TAstParseResult SqlToYql(const TString& query, const TTranslationSettings& settings, - NYql::TWarningRules* warningRules = nullptr, NYql::TStmtParseInfo* stmtParseInfo = nullptr); + NYql::TWarningRules* warningRules = nullptr, NYql::TStmtParseInfo* stmtParseInfo = nullptr, + TTranslationSettings* effectiveSettings = nullptr); google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NYql::TIssues& issues, size_t maxErrors, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr); ILexer::TPtr SqlLexer(const TString& query, NYql::TIssues& issues, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr); diff --git a/ydb/library/yql/sql/v1/sql_query.cpp b/ydb/library/yql/sql/v1/sql_query.cpp index bbf890ddb2f9..e06a4b39cc1d 100644 --- a/ydb/library/yql/sql/v1/sql_query.cpp +++ b/ydb/library/yql/sql/v1/sql_query.cpp @@ -1233,8 +1233,12 @@ bool TSqlQuery::Statement(TVector& blocks, const TRule_sql_stmt_core& } std::map features; - ParseViewOptions(features, node.GetRule_with_table_settings4()); - ParseViewQuery(features, node.GetRule_select_stmt6()); + if (!ParseViewOptions(features, node.GetRule_with_table_settings4())) { + return false; + } + if (!ParseViewQuery(features, node.GetRule_select_stmt6())) { + return false; + } const TString objectId = Id(node.GetRule_object_ref3().GetRule_id_or_at2(), *this).second; constexpr const char* TypeId = "VIEW"; diff --git a/ydb/library/yql/sql/v1/sql_translation.cpp b/ydb/library/yql/sql/v1/sql_translation.cpp index e06f5521d428..e94e225838a4 100644 --- a/ydb/library/yql/sql/v1/sql_translation.cpp +++ b/ydb/library/yql/sql/v1/sql_translation.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -53,21 +54,38 @@ TString CollectTokens(const TRule_select_stmt& selectStatement) { return tokenCollector.Tokens; } -NSQLTranslation::TTranslationSettings CreateViewTranslationSettings(const NSQLTranslation::TTranslationSettings& base) { - NSQLTranslation::TTranslationSettings settings; +bool RestoreContext( + TContext& ctx, const NSQLTranslation::TTranslationSettings& settings, const TString& contextRestorationQuery +) { + const TString queryName = "context restoration query"; + + const auto* ast = NSQLTranslationV1::SqlAST( + contextRestorationQuery, queryName, ctx.Issues, + settings.MaxErrors, settings.AnsiLexer, settings.Antlr4Parser, settings.TestAntlr4, settings.Arena + ); + if (!ast) { + return false; + } - settings.ClusterMapping = base.ClusterMapping; - settings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW; + TSqlQuery query(ctx, ctx.Settings.Mode, true); + auto node = query.Build(static_cast(*ast)); - return settings; + return node && node->Init(ctx, nullptr) && node->Translate(ctx); } -TNodePtr BuildViewSelect(const TRule_select_stmt& query, TContext& ctx) { - const auto viewTranslationSettings = CreateViewTranslationSettings(ctx.Settings); - TContext viewParsingContext(viewTranslationSettings, {}, ctx.Issues); - TSqlSelect select(viewParsingContext, viewTranslationSettings.Mode); +TNodePtr BuildViewSelect( + const TRule_select_stmt& selectQuery, + TContext& parentContext, + const TString& contextRestorationQuery +) { + TContext context(parentContext.Settings, {}, parentContext.Issues); + RestoreContext(context, context.Settings, contextRestorationQuery); + + context.Settings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW; + + TSqlSelect select(context, context.Settings.Mode); TPosition pos; - auto source = select.Build(query, pos); + auto source = select.Build(selectQuery, pos); if (!source) { return nullptr; } @@ -76,7 +94,7 @@ TNodePtr BuildViewSelect(const TRule_select_stmt& query, TContext& ctx) { std::move(source), false, false, - viewParsingContext.Scoped + context.Scoped ); } @@ -4594,12 +4612,32 @@ bool TSqlTranslation::ParseViewOptions(std::map& feature return true; } -bool TSqlTranslation::ParseViewQuery(std::map& features, - const TRule_select_stmt& query) { - const TString queryText = CollectTokens(query); - features["query_text"] = {Ctx.Pos(), queryText}; +bool TSqlTranslation::ParseViewQuery( + std::map& features, + const TRule_select_stmt& query +) { + TString queryText = CollectTokens(query); + TString contextRestorationQuery; + { + const auto& service = Ctx.Scoped->CurrService; + const auto& cluster = Ctx.Scoped->CurrCluster; + const auto effectivePathPrefix = Ctx.GetPrefixPath(service, cluster); + + // TO DO: capture all runtime pragmas in a similar fashion. + if (effectivePathPrefix != Ctx.Settings.PathPrefix) { + contextRestorationQuery = TStringBuilder() << "PRAGMA TablePathPrefix = \"" << effectivePathPrefix << "\";\n"; + } + + // TO DO: capture other compilation-affecting statements except USE. + if (cluster.GetLiteral() && *cluster.GetLiteral() != Ctx.Settings.DefaultCluster) { + contextRestorationQuery = TStringBuilder() << "USE " << *cluster.GetLiteral() << ";\n"; + } + } + features["query_text"] = { Ctx.Pos(), contextRestorationQuery + queryText }; - const auto viewSelect = BuildViewSelect(query, Ctx); + // AST is needed for ready-made validation of CREATE VIEW statement. + // Query is stored as plain text, not AST. + const auto viewSelect = BuildViewSelect(query, Ctx, contextRestorationQuery); if (!viewSelect) { return false; } diff --git a/ydb/services/metadata/manager/abstract.h b/ydb/services/metadata/manager/abstract.h index 16ce69160259..0b9c733960fd 100644 --- a/ydb/services/metadata/manager/abstract.h +++ b/ydb/services/metadata/manager/abstract.h @@ -14,6 +14,7 @@ #include #include +#include namespace NKikimr::NMetadata::NModifications { @@ -64,6 +65,7 @@ class IOperationsManager { YDB_ACCESSOR_DEF(TString, Database); using TActorSystemPtr = TActorSystem*; YDB_ACCESSOR_DEF(TActorSystemPtr, ActorSystem); + YDB_ACCESSOR_DEF(NSQLTranslation::TTranslationSettings, TranslationSettings); }; class TInternalModificationContext { diff --git a/ydb/services/metadata/manager/ya.make b/ydb/services/metadata/manager/ya.make index 79beeb811c81..1fd0b7871af9 100644 --- a/ydb/services/metadata/manager/ya.make +++ b/ydb/services/metadata/manager/ya.make @@ -21,6 +21,7 @@ PEERDIR( ydb/library/accessor ydb/library/actors/core ydb/library/table_creator + ydb/library/yql/sql/settings ydb/public/api/protos ydb/core/protos ydb/services/bg_tasks/abstract diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index 804ca76f1716..eab68f3adf91 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -7307,6 +7307,11 @@ "ColumnId": 3, "ColumnName": "QueryText", "ColumnType": "String" + }, + { + "ColumnId": 4, + "ColumnName": "CapturedContext", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -7315,7 +7320,8 @@ "Columns": [ 1, 2, - 3 + 3, + 4 ], "RoomID": 0, "Codec": 0, From 5b7bd0b4df93d83b9d74e77592f2dd685c3c7df7 Mon Sep 17 00:00:00 2001 From: Daniil Demin Date: Thu, 26 Sep 2024 08:12:51 +0000 Subject: [PATCH 2/2] fix build --- ydb/library/yql/sql/v1/sql_translation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/library/yql/sql/v1/sql_translation.cpp b/ydb/library/yql/sql/v1/sql_translation.cpp index e94e225838a4..9a79d1c07c53 100644 --- a/ydb/library/yql/sql/v1/sql_translation.cpp +++ b/ydb/library/yql/sql/v1/sql_translation.cpp @@ -61,7 +61,7 @@ bool RestoreContext( const auto* ast = NSQLTranslationV1::SqlAST( contextRestorationQuery, queryName, ctx.Issues, - settings.MaxErrors, settings.AnsiLexer, settings.Antlr4Parser, settings.TestAntlr4, settings.Arena + settings.MaxErrors, settings.AnsiLexer, settings.Arena ); if (!ast) { return false;