Capture TablePathPrefix (and other parts of the parser context) in CR…

…EATE VIEW (#8991) (#9780)

ydb/core/kqp/gateway/behaviour/view/manager.cpp

@@ -3,6 +3,7 @@
 #include <ydb/core/base/path.h>
 #include <ydb/core/kqp/gateway/actors/scheme.h>
 #include <ydb/core/kqp/gateway/utils/scheme_helpers.h>
+#include <ydb/core/kqp/provider/yql_kikimr_provider.h>
 #include <ydb/core/tx/tx_proxy/proxy.h>
 
 namespace NKikimr::NKqp {
@@ -11,13 +12,14 @@ namespace {
 
 using TYqlConclusionStatus = TViewManager::TYqlConclusionStatus;
 using TInternalModificationContext = TViewManager::TInternalModificationContext;
+using TExternalModificationContext = TViewManager::TExternalModificationContext;
 
 TString GetByKeyOrDefault(const NYql::TCreateObjectSettings& container, const TString& key) {
     const auto value = container.GetFeaturesExtractor().Extract(key);
     return value ? *value : TString{};
 }
 
-TYqlConclusionStatus CheckFeatureFlag(TInternalModificationContext& context) {
+TYqlConclusionStatus CheckFeatureFlag(const TInternalModificationContext& context) {
     auto* const actorSystem = context.GetExternalData().GetActorSystem();
     if (!actorSystem) {
         ythrow yexception() << "This place needs an actor system. Please contact internal support";
@@ -48,15 +50,16 @@ std::pair<TString, TString> SplitPathByObjectId(const TString& objectId) {
 
 void FillCreateViewProposal(NKikimrSchemeOp::TModifyScheme& modifyScheme,
                             const NYql::TCreateObjectSettings& settings,
-                            const TString& database) {
+                            const TExternalModificationContext& context) {
 
-    const auto pathPair = SplitPathByDb(settings.GetObjectId(), database);
+    const auto pathPair = SplitPathByDb(settings.GetObjectId(), context.GetDatabase());
     modifyScheme.SetWorkingDir(pathPair.first);
     modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateView);
 
     auto& viewDesc = *modifyScheme.MutableCreateView();
     viewDesc.SetName(pathPair.second);
     viewDesc.SetQueryText(GetByKeyOrDefault(settings, "query_text"));
+    NSQLTranslation::Serialize(context.GetTranslationSettings(), *viewDesc.MutableCapturedContext());
 
     if (!settings.GetFeaturesExtractor().IsFinished()) {
         ythrow TBadArgumentException() << "Unknown property: " << settings.GetFeaturesExtractor().GetRemainedParamsString();
@@ -92,20 +95,20 @@ NThreading::TFuture<TYqlConclusionStatus> SendSchemeRequest(TEvTxUserProxy::TEvP
 }
 
 NThreading::TFuture<TYqlConclusionStatus> CreateView(const NYql::TCreateObjectSettings& settings,
-                                                     TInternalModificationContext& context) {
+                                                     const TInternalModificationContext& context) {
     auto proposal = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
     proposal->Record.SetDatabaseName(context.GetExternalData().GetDatabase());
     if (context.GetExternalData().GetUserToken()) {
         proposal->Record.SetUserToken(context.GetExternalData().GetUserToken()->GetSerializedToken());
     }
     auto& schemeTx = *proposal->Record.MutableTransaction()->MutableModifyScheme();
-    FillCreateViewProposal(schemeTx, settings, context.GetExternalData().GetDatabase());
+    FillCreateViewProposal(schemeTx, settings, context.GetExternalData());
 
     return SendSchemeRequest(proposal.Release(), context.GetExternalData().GetActorSystem(), true);
 }
 
 NThreading::TFuture<TYqlConclusionStatus> DropView(const NYql::TDropObjectSettings& settings,
-                                                   TInternalModificationContext& context) {
+                                                   const TInternalModificationContext& context) {
     auto proposal = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
     proposal->Record.SetDatabaseName(context.GetExternalData().GetDatabase());
     if (context.GetExternalData().GetUserToken()) {
@@ -119,8 +122,8 @@ NThreading::TFuture<TYqlConclusionStatus> DropView(const NYql::TDropObjectSettin
 
 void PrepareCreateView(NKqpProto::TKqpSchemeOperation& schemeOperation,
                        const NYql::TObjectSettingsImpl& settings,
-                       TInternalModificationContext& context) {
-    FillCreateViewProposal(*schemeOperation.MutableCreateView(), settings, context.GetExternalData().GetDatabase());
+                       const TInternalModificationContext& context) {
+    FillCreateViewProposal(*schemeOperation.MutableCreateView(), settings, context.GetExternalData());
 }
 
 void PrepareDropView(NKqpProto::TKqpSchemeOperation& schemeOperation,

ydb/core/kqp/gateway/behaviour/view/ya.make

@@ -8,6 +8,7 @@ SRCS(
 PEERDIR(
     ydb/core/base
     ydb/core/kqp/gateway/actors
+    ydb/core/kqp/provider
     ydb/core/tx/tx_proxy
     ydb/services/metadata/abstract
     ydb/services/metadata/manager

ydb/core/kqp/gateway/kqp_metadata_loader.cpp

@@ -303,7 +303,7 @@ TTableMetadataResult GetViewMetadataResult(
   metadata->SchemaVersion = description.GetVersion();
   metadata->Kind = NYql::EKikimrTableKind::View;
   metadata->Attributes = schemeEntry.Attributes;
-  metadata->ViewPersistedData = {description.GetQueryText()};
+  metadata->ViewPersistedData = {description.GetQueryText(), description.GetCapturedContext()};
 
   return builtResult;
 }

ydb/core/kqp/host/kqp_gateway_proxy.cpp

@@ -1266,6 +1266,7 @@ class TKqpGatewayProxy : public IKikimrGateway {
             if (SessionCtx->GetUserToken()) {
                 context.SetUserToken(*SessionCtx->GetUserToken());
             }
+            context.SetTranslationSettings(SessionCtx->Query().TranslationSettings);
 
             auto& phyTx = phyTxRemover.Capture(SessionCtx->Query().PreparingQuery->MutablePhysicalQuery());
             phyTx.SetType(NKqpProto::TKqpPhyTx::TYPE_SCHEME);

ydb/core/kqp/host/kqp_host.cpp

@@ -1228,7 +1228,19 @@ class TKqpHost : public IKqpHost {
                 .SetQueryParameters(query.ParameterTypes)
                 .SetApplicationName(ApplicationName)
                 .SetIsEnablePgSyntax(SessionCtx->Config().FeatureFlags.GetEnablePgSyntax());
-            auto astRes = ParseQuery(query.Text, isSql, sqlVersion, TypesCtx->DeprecatedSQL, ctx, settingsBuilder, result.KeepInCache, result.CommandTagName);
+            NSQLTranslation::TTranslationSettings effectiveSettings;
+            auto astRes = ParseQuery(
+                query.Text,
+                isSql,
+                sqlVersion,
+                TypesCtx->DeprecatedSQL,
+                ctx,
+                settingsBuilder,
+                result.KeepInCache,
+                result.CommandTagName,
+                &effectiveSettings
+            );
+            SessionCtx->Query().TranslationSettings = std::move(effectiveSettings);
             if (astRes.ActualSyntaxType == NYql::ESyntaxType::Pg) {
                 SessionCtx->Config().IndexAutoChooserMode = NKikimrConfig::TTableServiceConfig_EIndexAutoChooseMode::TTableServiceConfig_EIndexAutoChooseMode_MAX_USED_PREFIX;
             }

ydb/core/kqp/host/kqp_translate.cpp

@@ -54,7 +54,7 @@ NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type) {
             YQL_ENSURE(false, "Unexpected query type: " << type);
     }
 }
-           
+
 NSQLTranslation::TTranslationSettings TKqpTranslationSettingsBuilder::Build(NYql::TExprContext& ctx) {
     NSQLTranslation::TTranslationSettings settings;
     settings.PgParser = UsePgParser && *UsePgParser;
@@ -154,13 +154,14 @@ NSQLTranslation::TTranslationSettings TKqpTranslationSettingsBuilder::Build(NYql
 }
 
 NYql::TAstParseResult ParseQuery(const TString& queryText, bool isSql, TMaybe<ui16>& sqlVersion, bool& deprecatedSQL,
-        NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName) {
+        NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName,
+        NSQLTranslation::TTranslationSettings* effectiveSettings) {
     NYql::TAstParseResult astRes;
     settingsBuilder.SetSqlVersion(sqlVersion);
     if (isSql) {
         auto settings = settingsBuilder.Build(ctx);
         NYql::TStmtParseInfo stmtParseInfo;
-        auto ast = NSQLTranslation::SqlToYql(queryText, settings, nullptr, &stmtParseInfo);
+        auto ast = NSQLTranslation::SqlToYql(queryText, settings, nullptr, &stmtParseInfo, effectiveSettings);
         deprecatedSQL = (ast.ActualSyntaxType == NYql::ESyntaxType::YQLv0);
         sqlVersion = ast.ActualSyntaxType == NYql::ESyntaxType::YQLv1 ? 1 : 0;
         keepInCache = stmtParseInfo.KeepInCache;

ydb/core/kqp/host/kqp_translate.h

@@ -91,7 +91,8 @@ NSQLTranslation::EBindingsMode RemapBindingsMode(NKikimrConfig::TTableServiceCon
 NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type);
 
 NYql::TAstParseResult ParseQuery(const TString& queryText, bool isSql, TMaybe<ui16>& sqlVersion, bool& deprecatedSQL,
-    NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName);
+    NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName,
+    NSQLTranslation::TTranslationSettings* effectiveSettings = nullptr);
 
 TVector<TQueryAst> ParseStatements(const TString& queryText, const TMaybe<Ydb::Query::Syntax>& syntax, bool isSql, TKqpTranslationSettingsBuilder& settingsBuilder, bool perStatementExecution);
 

ydb/core/kqp/provider/rewrite_io_utils.cpp

@@ -1,6 +1,7 @@
 #include "rewrite_io_utils.h"
 
 #include <ydb/core/kqp/provider/yql_kikimr_expr_nodes.h>
+#include <ydb/core/kqp/provider/yql_kikimr_provider.h>
 #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
 #include <ydb/library/yql/core/yql_expr_optimize.h>
 #include <ydb/library/yql/providers/common/provider/yql_provider.h>
@@ -16,16 +17,17 @@ using namespace NNodes;
 constexpr const char* QueryGraphNodeSignature = "SavedQueryGraph";
 
 TExprNode::TPtr CompileViewQuery(
-    const TString& query,
     TExprContext& ctx,
     NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder,
-    IModuleResolver::TPtr moduleResolver
+    IModuleResolver::TPtr moduleResolver,
+    const TViewPersistedData& viewData
 ) {
     auto translationSettings = settingsBuilder.Build(ctx);
     translationSettings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW;
+    NSQLTranslation::Deserialize(viewData.CapturedContext, translationSettings);
 
     TAstParseResult queryAst;
-    queryAst = NSQLTranslation::SqlToYql(query, translationSettings);
+    queryAst = NSQLTranslation::SqlToYql(viewData.QueryText, translationSettings);
 
     ctx.IssueManager.AddIssues(queryAst.Issues);
     if (!queryAst.IsOk()) {
@@ -116,9 +118,9 @@ TExprNode::TPtr FindTopLevelRead(const TExprNode::TPtr& queryGraph) {
 TExprNode::TPtr RewriteReadFromView(
     const TExprNode::TPtr& node,
     TExprContext& ctx,
-    const TString& query,
     NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder,
-    IModuleResolver::TPtr moduleResolver
+    IModuleResolver::TPtr moduleResolver,
+    const TViewPersistedData& viewData
 ) {
     YQL_PROFILE_FUNC(DEBUG);
 
@@ -127,7 +129,7 @@ TExprNode::TPtr RewriteReadFromView(
 
     TExprNode::TPtr queryGraph = FindSavedQueryGraph(readNode.Ptr());
     if (!queryGraph) {
-        queryGraph = CompileViewQuery(query, ctx, settingsBuilder, moduleResolver);
+        queryGraph = CompileViewQuery(ctx, settingsBuilder, moduleResolver, viewData);
         if (!queryGraph) {
             ctx.AddError(TIssue(ctx.GetPosition(readNode.Pos()),
                          "The query stored in the view cannot be compiled."));
@@ -151,4 +153,4 @@ TExprNode::TPtr RewriteReadFromView(
     return Build<TCoLeft>(ctx, node->Pos()).Input(topLevelRead).Done().Ptr();
 }
 
-}
+}

ydb/core/kqp/provider/rewrite_io_utils.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <ydb/core/kqp/host/kqp_translate.h>
+#include <ydb/core/kqp/provider/yql_kikimr_gateway.h>
 #include <ydb/library/yql/ast/yql_expr.h>
 
 namespace NYql {
@@ -10,9 +11,9 @@ TExprNode::TPtr FindTopLevelRead(const TExprNode::TPtr& queryGraph);
 TExprNode::TPtr RewriteReadFromView(
     const TExprNode::TPtr& node,
     TExprContext& ctx,
-    const TString& query,
     NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder,
-    IModuleResolver::TPtr moduleResolver
+    IModuleResolver::TPtr moduleResolver,
+    const TViewPersistedData& viewData
 );
 
-}
+}

ydb/core/kqp/provider/yql_kikimr_datasink.cpp

@@ -176,9 +176,9 @@ class TKiSinkIntentDeterminationTransformer: public TKiSinkVisitorTransformer {
     }
 
     TStatus HandleDropObject(TKiDropObject node, TExprContext& ctx) override {
-        ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder()
-            << "DropObject is not yet implemented for intent determination transformer"));
-        return TStatus::Error;
+        Y_UNUSED(node);
+        Y_UNUSED(ctx);
+        return TStatus::Ok;
     }
 
     TStatus HandleCreateGroup(TKiCreateGroup node, TExprContext& ctx) override {

ydb/core/kqp/provider/yql_kikimr_datasource.cpp

@@ -771,16 +771,17 @@ class TKikimrDataSource : public TDataProviderBase {
                     .Repeat(TExprStep::LoadTablesMetadata)
                     .Repeat(TExprStep::RewriteIO);
 
-                const auto& query = tableDesc.Metadata->ViewPersistedData.QueryText;
+                const auto& viewData = tableDesc.Metadata->ViewPersistedData;
+
                 NKqp::TKqpTranslationSettingsBuilder settingsBuilder(
                     SessionCtx->Query().Type,
                     SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(),
                     cluster,
-                    query,
+                    viewData.QueryText,
                     SessionCtx->Config().BindingsMode,
                     GUCSettings
                 );
-                return RewriteReadFromView(node, ctx, query, settingsBuilder, Types.Modules);
+                return RewriteReadFromView(node, ctx, settingsBuilder, Types.Modules, viewData);
             }
         }
 

ydb/core/kqp/provider/yql_kikimr_gateway.h

@@ -22,6 +22,7 @@
 #include <ydb/core/protos/flat_scheme_op.pb.h>
 #include <ydb/core/protos/kqp.pb.h>
 #include <ydb/core/protos/kqp_stats.pb.h>
+#include <ydb/core/protos/yql_translation_settings.pb.h>
 #include <ydb/core/scheme/scheme_types_proto.h>
 
 #include <library/cpp/json/json_reader.h>
@@ -406,6 +407,7 @@ enum EMetaSerializationType : ui64 {
 
 struct TViewPersistedData {
     TString QueryText;
+    NYql::NProto::TTranslationSettings CapturedContext;
 };
 
 struct TKikimrTableMetadata : public TThrRefBase {

ydb/core/kqp/provider/yql_kikimr_provider.cpp

@@ -964,3 +964,36 @@ TCoNameValueTupleList TKiExecDataQuerySettings::BuildNode(TExprContext& ctx, TPo
 }
 
 } // namespace NYql
+
+namespace NSQLTranslation {
+
+void Serialize(const TTranslationSettings& settings, NYql::NProto::TTranslationSettings& serializedSettings) {
+    serializedSettings.SetPathPrefix(settings.PathPrefix);
+    serializedSettings.SetSyntaxVersion(settings.SyntaxVersion);
+    serializedSettings.SetAnsiLexer(settings.AnsiLexer);
+    serializedSettings.SetPgParser(settings.PgParser);
+
+    auto* pragmas = serializedSettings.MutablePragmas();
+    pragmas->Clear();
+    pragmas->Add(settings.Flags.begin(), settings.Flags.end());
+}
+
+void Deserialize(const NYql::NProto::TTranslationSettings& serializedSettings, TTranslationSettings& settings) {
+    #define DeserializeSetting(settingName) \
+        if (serializedSettings.Has##settingName()) { \
+            settings.settingName = serializedSettings.Get##settingName(); \
+        }
+
+        DeserializeSetting(PathPrefix);
+        DeserializeSetting(SyntaxVersion);
+        DeserializeSetting(AnsiLexer);
+        DeserializeSetting(PgParser);
+
+    #undef DeserializeSetting
+
+    // overwrite existing pragmas
+    settings.Flags.clear();
+    settings.Flags.insert(serializedSettings.GetPragmas().begin(), serializedSettings.GetPragmas().end());
+}
+
+}

ydb/core/kqp/provider/yql_kikimr_provider.h

@@ -124,6 +124,8 @@ struct TKikimrQueryContext : TThrRefBase {
     // we do not want add extra life time for query context here
     std::shared_ptr<NKikimr::NGRpcService::IRequestCtxMtSafe> RpcCtx;
 
+    NSQLTranslation::TTranslationSettings TranslationSettings;
+
     void Reset() {
         PrepareOnly = false;
         SuppressDdlChecks = false;
@@ -142,6 +144,7 @@ struct TKikimrQueryContext : TThrRefBase {
 
         RlPath.Clear();
         RpcCtx.reset();
+        TranslationSettings = NSQLTranslation::TTranslationSettings();
     }
 };
 
@@ -578,3 +581,10 @@ TIntrusivePtr<IDataProvider> CreateKikimrDataSink(
     TIntrusivePtr<IKikimrQueryExecutor> queryExecutor);
 
 } // namespace NYql
+
+namespace NSQLTranslation {
+
+void Serialize(const TTranslationSettings& settings, NYql::NProto::TTranslationSettings& serializedSettings);
+void Deserialize(const NYql::NProto::TTranslationSettings& serializedSettings, TTranslationSettings& settings);
+
+}