From 40c674f2fec5854508384f97dd4f2de619ec9754 Mon Sep 17 00:00:00 2001 From: Polina Volosnikova Date: Fri, 26 Jan 2024 16:23:18 +0000 Subject: [PATCH 1/5] KIKIMR-20543: ddl+dml in query service --- ydb/core/kqp/common/compilation/events.h | 19 ++- ydb/core/kqp/common/kqp_lwtrace_probes.h | 3 + .../kqp/compile_service/kqp_compile_actor.cpp | 64 ++++++--- .../compile_service/kqp_compile_service.cpp | 126 ++++++++++++------ .../kqp/compile_service/kqp_compile_service.h | 5 +- ydb/core/kqp/host/kqp_translate.cpp | 54 +++++++- ydb/core/kqp/host/kqp_translate.h | 5 +- ydb/core/kqp/provider/yql_kikimr_settings.h | 1 + .../kqp/session_actor/kqp_query_state.cpp | 20 ++- ydb/core/kqp/session_actor/kqp_query_state.h | 40 ++++++ .../kqp/session_actor/kqp_session_actor.cpp | 32 ++++- ydb/core/kqp/ut/pg/kqp_pg_ut.cpp | 27 ++++ ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp | 124 +++++++++++++++++ ydb/core/protos/feature_flags.proto | 2 +- ydb/core/protos/table_service_config.proto | 3 + .../yql/parser/pg_wrapper/interface/parser.h | 1 + .../parser/pg_wrapper/interface/raw_parser.h | 2 + ydb/library/yql/parser/pg_wrapper/parser.cpp | 29 ++++ ydb/library/yql/sql/pg/pg_sql.cpp | 45 ++++++- ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp | 6 + .../yql/sql/settings/translation_settings.h | 1 + ydb/library/yql/sql/sql.cpp | 52 +++++++- ydb/library/yql/sql/sql.h | 2 + ydb/library/yql/sql/v1/sql.cpp | 98 ++++++++++++++ ydb/library/yql/sql/v1/sql.h | 1 + ydb/library/yql/sql/v1/sql_query.cpp | 70 ++++++++++ ydb/library/yql/sql/v1/sql_query.h | 1 + ydb/library/yql/sql/v1/sql_ut.cpp | 11 ++ 28 files changed, 761 insertions(+), 83 deletions(-) diff --git a/ydb/core/kqp/common/compilation/events.h b/ydb/core/kqp/common/compilation/events.h index c1272f74a833..a1a76be65b25 100644 --- a/ydb/core/kqp/common/compilation/events.h +++ b/ydb/core/kqp/common/compilation/events.h @@ -14,14 +14,15 @@ namespace NKikimr::NKqp::NPrivateEvents { struct TEvCompileRequest: public TEventLocal { TEvCompileRequest(const TIntrusiveConstPtr& userToken, const TMaybe& uid, - TMaybe&& query, bool keepInCache, TInstant deadline, + TMaybe&& query, bool keepInCache, bool canDevideIntoStatements, TInstant deadline, TKqpDbCountersPtr dbCounters, std::shared_ptr> intrestedInResult, const TIntrusivePtr& userRequestContext, NLWTrace::TOrbit orbit = {}, - TKqpTempTablesState::TConstPtr tempTablesState = nullptr, bool collectDiagnostics = false) + TKqpTempTablesState::TConstPtr tempTablesState = nullptr, bool collectDiagnostics = false, TMaybe queryAst = Nothing()) : UserToken(userToken) , Uid(uid) , Query(std::move(query)) , KeepInCache(keepInCache) + , CanDevideIntoStatements(canDevideIntoStatements) , Deadline(deadline) , DbCounters(dbCounters) , UserRequestContext(userRequestContext) @@ -29,6 +30,7 @@ struct TEvCompileRequest: public TEventLocal Uid; TMaybe Query; bool KeepInCache = false; + bool CanDevideIntoStatements = false; // it is allowed for local event to use absolute time (TInstant) instead of time interval (TDuration) TInstant Deadline; TKqpDbCountersPtr DbCounters; @@ -49,6 +52,8 @@ struct TEvCompileRequest: public TEventLocal> IntrestedInResult; bool CollectDiagnostics = false; + + TMaybe QueryAst; }; struct TEvRecompileRequest: public TEventLocal { @@ -99,12 +104,14 @@ struct TEvCompileResponse: public TEventLocal { - TEvParseResponse(const TKqpQueryId& query, TMaybe astResult) - : AstResult(std::move(astResult)) - , Query(query) {} + TEvParseResponse(const TKqpQueryId& query, TVector astStatements, NLWTrace::TOrbit orbit = {}) + : AstStatements(std::move(astStatements)) + , Query(query) + , Orbit(std::move(orbit)) {} - TMaybe AstResult; + TVector AstStatements; TKqpQueryId Query; + NLWTrace::TOrbit Orbit; }; struct TEvCompileInvalidateRequest: public TEventLocal { TKqpDbCountersPtr dbCounters, std::optional federatedQuerySetup, const TIntrusivePtr& userRequestContext, NWilson::TTraceId traceId, TKqpTempTablesState::TConstPtr tempTablesState, bool collectFullDiagnostics, - ECompileActorAction compileAction, TMaybe astResult) + bool canDevideIntoStatements, ECompileActorAction compileAction, TMaybe queryAst) : Owner(owner) , ModuleResolverState(moduleResolverState) , Counters(counters) , FederatedQuerySetup(federatedQuerySetup) , Uid(uid) , QueryId(queryId) - , QueryRef(QueryId.Text, QueryId.QueryParameterTypes, astResult) + , QueryRef(QueryId.Text, QueryId.QueryParameterTypes, queryAst) , UserToken(userToken) , DbCounters(dbCounters) , Config(MakeIntrusive()) @@ -70,8 +70,9 @@ class TKqpCompileActor : public TActorBootstrapped { , CompileActorSpan(TWilsonKqp::CompileActor, std::move(traceId), "CompileActor") , TempTablesState(std::move(tempTablesState)) , CollectFullDiagnostics(collectFullDiagnostics) + , CanDevideIntoStatements(canDevideIntoStatements) , CompileAction(compileAction) - , AstResult(std::move(astResult)) + , QueryAst(std::move(queryAst)) { Config->Init(kqpSettings->DefaultSettings.GetDefaultSettings(), QueryId.Cluster, kqpSettings->Settings, false); @@ -127,26 +128,22 @@ class TKqpCompileActor : public TActorBootstrapped { } private: - void SetQueryAst(const TActorContext &ctx) { + + TVector GetAstStatements(const TActorContext &ctx) { TString cluster = QueryId.Cluster; TString kqpTablePathPrefix = Config->_KqpTablePathPrefix.Get().GetRef(); ui16 kqpYqlSyntaxVersion = Config->_KqpYqlSyntaxVersion.Get().GetRef(); NSQLTranslation::EBindingsMode bindingsMode = Config->BindingsMode; bool isEnableExternalDataSources = AppData(ctx)->FeatureFlags.GetEnableExternalDataSources(); bool isEnablePgConstsToParams = Config->EnablePgConstsToParams; + bool perStatement = Config->EnableQueriesPerStatement && CanDevideIntoStatements; - auto astResult = ParseQuery(ConvertType(QueryId.Settings.QueryType), QueryId.Settings.Syntax, QueryId.Text, QueryId.QueryParameterTypes, QueryId.IsSql(), cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, isEnablePgConstsToParams); - YQL_ENSURE(astResult.Ast); - if (astResult.Ast->IsOk()) { - AstResult = std::move(astResult); - } + return SqlToAstStatements(ConvertType(QueryId.Settings.QueryType), QueryId.Settings.Syntax, QueryId.Text, QueryId.QueryParameterTypes, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, isEnablePgConstsToParams, QueryId.IsSql(), perStatement); } void StartParsing(const TActorContext &ctx) { - SetQueryAst(ctx); - Become(&TKqpCompileActor::CompileState); - ReplyParseResult(ctx); + ReplyParseResult(ctx, GetAstStatements(ctx)); } void StartCompilation(const TActorContext &ctx) { @@ -352,15 +349,37 @@ class TKqpCompileActor : public TActorBootstrapped { << ", at state:" << state); } - void ReplyParseResult(const TActorContext &ctx) { + void ReplyParseResult(const TActorContext &ctx, TVector astStatements) { Y_UNUSED(ctx); + + if (astStatements.empty()) { + NYql::TIssue issue(NYql::TPosition(), "Parsing result of query is empty"); + ReplyError(Ydb::StatusIds::GENERIC_ERROR, {issue}); + } + + for (size_t statementId = 0; statementId < astStatements.size(); ++statementId) { + if (!astStatements[statementId].Ast || !astStatements[statementId].Ast->IsOk() || !astStatements[statementId].Ast->Root) { + ALOG_ERROR(NKikimrServices::KQP_COMPILE_ACTOR, "Get parsing result with error" + << ", self: " << SelfId() + << ", owner: " << Owner + << ", statement id: " << statementId); + + NYql::TIssue issue(NYql::TPosition(), "Internal error while parsing query."); + for (const auto& i : astStatements[statementId].Ast->Issues) { + issue.AddSubIssue(MakeIntrusive(i)); + } + + ReplyError(Ydb::StatusIds::GENERIC_ERROR, {issue}); + return; + } + } + ALOG_DEBUG(NKikimrServices::KQP_COMPILE_ACTOR, "Send parsing result" << ", self: " << SelfId() << ", owner: " << Owner - << (AstResult && AstResult->Ast->IsOk() ? ", parsing is successful" : ", parsing is not successful")); + << ", statements size: " << astStatements.size()); - auto responseEv = MakeHolder(QueryId, std::move(AstResult)); - AstResult = Nothing(); + auto responseEv = MakeHolder(QueryId, astStatements); Send(Owner, responseEv.Release()); Counters->ReportCompileFinish(DbCounters); @@ -379,8 +398,8 @@ class TKqpCompileActor : public TActorBootstrapped { KqpCompileResult->PreparedQuery = preparedQueryHolder; KqpCompileResult->AllowCache = CanCacheQuery(KqpCompileResult->PreparedQuery->GetPhysicalQuery()); - if (AstResult) { - KqpCompileResult->Ast = AstResult->Ast; + if (QueryAst) { + KqpCompileResult->Ast = QueryAst->Ast; } } @@ -482,8 +501,9 @@ class TKqpCompileActor : public TActorBootstrapped { TKqpTempTablesState::TConstPtr TempTablesState; bool CollectFullDiagnostics; + bool CanDevideIntoStatements; ECompileActorAction CompileAction; - TMaybe AstResult; + TMaybe QueryAst; }; void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConfig& serviceConfig) { @@ -512,6 +532,7 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf kqpConfig.IndexAutoChooserMode = serviceConfig.GetIndexAutoChooseMode(); kqpConfig.EnablePgConstsToParams = serviceConfig.GetEnablePgConstsToParams() && serviceConfig.GetEnableAstCache(); kqpConfig.ExtractPredicateRangesLimit = serviceConfig.GetExtractPredicateRangesLimit(); + kqpConfig.EnableQueriesPerStatement = serviceConfig.GetEnableQueriesPerStatement(); if (const auto limit = serviceConfig.GetResourceManager().GetMkqlHeavyProgramMemoryLimit()) { kqpConfig._KqpYqlCombinerMemoryLimit = std::max(1_GB, limit - (limit >> 2U)); @@ -527,14 +548,15 @@ IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstP std::optional federatedQuerySetup, TKqpDbCountersPtr dbCounters, const TIntrusivePtr& userRequestContext, NWilson::TTraceId traceId, TKqpTempTablesState::TConstPtr tempTablesState, - ECompileActorAction compileAction, TMaybe astResult, bool collectFullDiagnostics) + ECompileActorAction compileAction, TMaybe queryAst, bool collectFullDiagnostics, + bool canDevideIntoStatements) { return new TKqpCompileActor(owner, kqpSettings, tableServiceConfig, queryServiceConfig, metadataProviderConfig, moduleResolverState, counters, uid, query, userToken, dbCounters, federatedQuerySetup, userRequestContext, std::move(traceId), std::move(tempTablesState), collectFullDiagnostics, - compileAction, std::move(astResult)); + canDevideIntoStatements, compileAction, std::move(queryAst)); } } // namespace NKqp diff --git a/ydb/core/kqp/compile_service/kqp_compile_service.cpp b/ydb/core/kqp/compile_service/kqp_compile_service.cpp index f40d8348b6f5..43db6a69b6b7 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_service.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_service.cpp @@ -51,8 +51,10 @@ class TKqpQueryCache { AstIndex.emplace(GetQueryIdWithAst(*compileResult->Query, *compileResult->Ast), compileResult->Uid); } - bool Insert(const TKqpCompileResult::TConstPtr& compileResult, bool isEnableAstCache) { - InsertQuery(compileResult); + bool Insert(const TKqpCompileResult::TConstPtr& compileResult, bool isEnableAstCache, bool isPerStatement) { + if (!isPerStatement) { + InsertQuery(compileResult); + } if (isEnableAstCache && compileResult->Ast) { InsertAst(compileResult); } @@ -81,7 +83,6 @@ class TKqpQueryCache { } Y_ABORT_UNLESS(List.GetSize() == Index.size()); - Y_ABORT_UNLESS(List.GetSize() == QueryIndex.size()); return removedItem != nullptr; } @@ -168,7 +169,6 @@ class TKqpQueryCache { Index.erase(it); Y_ABORT_UNLESS(List.GetSize() == Index.size()); - Y_ABORT_UNLESS(List.GetSize() == QueryIndex.size()); return true; } @@ -189,7 +189,6 @@ class TKqpQueryCache { } Y_ABORT_UNLESS(List.GetSize() == Index.size()); - Y_ABORT_UNLESS(List.GetSize() == QueryIndex.size()); return prevSize - Size(); } @@ -233,18 +232,19 @@ class TKqpQueryCache { }; struct TKqpCompileRequest { - TKqpCompileRequest(const TActorId& sender, const TString& uid, TKqpQueryId query, bool keepInCache, + TKqpCompileRequest(const TActorId& sender, const TString& uid, TKqpQueryId query, bool keepInCache, bool canDevideIntoStatements, const TIntrusiveConstPtr& userToken, const TInstant& deadline, TKqpDbCountersPtr dbCounters, ui64 cookie, std::shared_ptr> intrestedInResult, const TIntrusivePtr& userRequestContext, NLWTrace::TOrbit orbit = {}, NWilson::TSpan span = {}, TKqpTempTablesState::TConstPtr tempTablesState = {}, ECompileActorAction action = ECompileActorAction::COMPILE, - TMaybe astResult = {}) + TMaybe queryAst = {}) : Sender(sender) , Query(std::move(query)) , Uid(uid) , KeepInCache(keepInCache) + , CanDevideIntoStatements(canDevideIntoStatements) , UserToken(userToken) , Deadline(deadline) , DbCounters(dbCounters) @@ -255,13 +255,14 @@ struct TKqpCompileRequest { , TempTablesState(std::move(tempTablesState)) , IntrestedInResult(std::move(intrestedInResult)) , Action(action) - , AstResult(std::move(astResult)) + , QueryAst(std::move(queryAst)) {} TActorId Sender; TKqpQueryId Query; TString Uid; bool KeepInCache = false; + bool CanDevideIntoStatements = false; TIntrusiveConstPtr UserToken; TInstant Deadline; TKqpDbCountersPtr DbCounters; @@ -274,7 +275,7 @@ struct TKqpCompileRequest { TKqpTempTablesState::TConstPtr TempTablesState; std::shared_ptr> IntrestedInResult; ECompileActorAction Action; - TMaybe AstResult; + TMaybe QueryAst; bool IsIntrestedInResult() const { return IntrestedInResult->load(); @@ -637,11 +638,15 @@ class TKqpCompileService : public TActorBootstrapped { ev->Get()->Query ? ev->Get()->Query->UserSid : 0); TKqpCompileRequest compileRequest(ev->Sender, CreateGuidAsString(), std::move(*request.Query), - request.KeepInCache, request.UserToken, request.Deadline, dbCounters, + request.KeepInCache, request.CanDevideIntoStatements, request.UserToken, request.Deadline, dbCounters, ev->Cookie, std::move(ev->Get()->IntrestedInResult), ev->Get()->UserRequestContext, std::move(ev->Get()->Orbit), std::move(compileServiceSpan), std::move(ev->Get()->TempTablesState), TableServiceConfig.GetEnableAstCache() ? ECompileActorAction::PARSE : ECompileActorAction::COMPILE); + if (TableServiceConfig.GetEnableAstCache() && request.QueryAst) { + return CompileByAst(*request.QueryAst, compileRequest, ctx); + } + if (!RequestsQueue.Enqueue(std::move(compileRequest))) { Counters->ReportCompileRequestRejected(dbCounters); @@ -694,7 +699,7 @@ class TKqpCompileService : public TActorBootstrapped { NWilson::TSpan compileServiceSpan(TWilsonKqp::CompileService, ev->Get() ? std::move(ev->TraceId) : NWilson::TTraceId(), "CompileService"); TKqpCompileRequest compileRequest(ev->Sender, request.Uid, compileResult ? *compileResult->Query : *request.Query, - true, request.UserToken, request.Deadline, dbCounters, + true, false, request.UserToken, request.Deadline, dbCounters, ev->Cookie, std::move(ev->Get()->IntrestedInResult), ev->Get()->UserRequestContext, ev->Get() ? std::move(ev->Get()->Orbit) : NLWTrace::TOrbit(), @@ -751,13 +756,14 @@ class TKqpCompileService : public TActorBootstrapped { << ", compileActor: " << ev->Sender); bool keepInCache = compileRequest.KeepInCache && compileResult->AllowCache; + bool isPerStatement = TableServiceConfig.GetEnableAstCache() && compileRequest.QueryAst; bool hasTempTablesNameClashes = HasTempTablesNameClashes(compileResult, compileRequest.TempTablesState, true); try { if (compileResult->Status == Ydb::StatusIds::SUCCESS) { if (!hasTempTablesNameClashes) { - UpdateQueryCache(compileResult, keepInCache); + UpdateQueryCache(compileResult, keepInCache, isPerStatement); } if (ev->Get()->ReplayMessage) { @@ -838,43 +844,44 @@ class TKqpCompileService : public TActorBootstrapped { return compileResult->PreparedQuery->HasTempTables(tempTablesState, withSessionId); } - void UpdateQueryCache(TKqpCompileResult::TConstPtr compileResult, bool keepInCache) { + void UpdateQueryCache(TKqpCompileResult::TConstPtr compileResult, bool keepInCache, bool isPerStatement) { if (QueryCache.FindByUid(compileResult->Uid, false)) { QueryCache.Replace(compileResult); } else if (keepInCache) { - if (QueryCache.Insert(compileResult, TableServiceConfig.GetEnableAstCache())) { + if (QueryCache.Insert(compileResult, TableServiceConfig.GetEnableAstCache(), isPerStatement)) { Counters->CompileQueryCacheEvicted->Inc(); } if (compileResult->Query && compileResult->Query->Settings.IsPrepareQuery) { - if (InsertPreparingQuery(compileResult, true)) { + if (InsertPreparingQuery(compileResult, true, isPerStatement)) { Counters->CompileQueryCacheEvicted->Inc(); }; } } } - void Handle(TEvKqp::TEvParseResponse::TPtr& ev, const TActorContext& ctx) { - auto& parseResult = ev->Get()->AstResult; - auto& query = ev->Get()->Query; - auto compileRequest = RequestsQueue.FinishActiveRequest(query); - if (parseResult && parseResult->Ast->IsOk()) { - auto compileResult = QueryCache.FindByAst(query, *parseResult->Ast, compileRequest.KeepInCache); - if (HasTempTablesNameClashes(compileResult, compileRequest.TempTablesState)) { - compileResult = nullptr; - } - if (compileResult) { - Counters->ReportQueryCacheHit(compileRequest.DbCounters, true); + void CompileByAst(const TQueryAst& queryAst, TKqpCompileRequest& compileRequest, const TActorContext& ctx) { + YQL_ENSURE(queryAst.Ast); + YQL_ENSURE(queryAst.Ast->IsOk()); + YQL_ENSURE(queryAst.Ast->Root); + auto compileResult = QueryCache.FindByAst(compileRequest.Query, *queryAst.Ast, compileRequest.KeepInCache); + + if (HasTempTablesNameClashes(compileResult, compileRequest.TempTablesState)) { + compileResult = nullptr; + } - LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Served query from cache from ast" - << ", sender: " << compileRequest.Sender - << ", queryUid: " << compileResult->Uid); + if (compileResult) { + Counters->ReportQueryCacheHit(compileRequest.DbCounters, true); - compileResult->Ast->PgAutoParamValues = std::move(parseResult->Ast->PgAutoParamValues); + LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Served query from cache from ast" + << ", sender: " << compileRequest.Sender + << ", queryUid: " << compileResult->Uid); - ReplyFromCache(compileRequest.Sender, compileResult, ctx, compileRequest.Cookie, std::move(compileRequest.Orbit), std::move(compileRequest.CompileServiceSpan)); - return; - } + compileResult->Ast->PgAutoParamValues = std::move(queryAst.Ast->PgAutoParamValues); + + ReplyFromCache(compileRequest.Sender, compileResult, ctx, compileRequest.Cookie, std::move(compileRequest.Orbit), std::move(compileRequest.CompileServiceSpan)); + return; } + Counters->ReportQueryCacheHit(compileRequest.DbCounters, false); LWTRACK(KqpCompileServiceEnqueued, @@ -882,30 +889,43 @@ class TKqpCompileService : public TActorBootstrapped { compileRequest.Query.UserSid); compileRequest.Action = ECompileActorAction::COMPILE; - compileRequest.AstResult = std::move(parseResult); + compileRequest.QueryAst = std::move(queryAst); if (!RequestsQueue.Enqueue(std::move(compileRequest))) { Counters->ReportCompileRequestRejected(compileRequest.DbCounters); LOG_WARN_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Requests queue size limit exceeded" - << ", sender: " << ev->Sender + << ", sender: " << compileRequest.Sender << ", queueSize: " << RequestsQueue.Size()); NYql::TIssue issue(NYql::TPosition(), TStringBuilder() << "Exceeded maximum number of requests in compile service queue."); - ReplyError(ev->Sender, "", Ydb::StatusIds::OVERLOADED, {issue}, ctx, compileRequest.Cookie, std::move(compileRequest.Orbit), std::move(compileRequest.CompileServiceSpan)); + ReplyError(compileRequest.Sender, "", Ydb::StatusIds::OVERLOADED, {issue}, ctx, compileRequest.Cookie, std::move(compileRequest.Orbit), std::move(compileRequest.CompileServiceSpan)); return; } LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Added request to queue" - << ", sender: " << ev->Sender + << ", sender: " << compileRequest.Sender << ", queueSize: " << RequestsQueue.Size()); ProcessQueue(ctx); } + void Handle(TEvKqp::TEvParseResponse::TPtr& ev, const TActorContext& ctx) { + auto& astStatements = ev->Get()->AstStatements; + YQL_ENSURE(astStatements.size()); + auto& query = ev->Get()->Query; + auto compileRequest = RequestsQueue.FinishActiveRequest(query); + if (astStatements.size() > 1) { + ReplyQueryStatements(compileRequest.Sender, astStatements, query, ctx, compileRequest.Cookie, std::move(compileRequest.Orbit), std::move(compileRequest.CompileServiceSpan)); + return; + } + + CompileByAst(astStatements.front(), compileRequest, ctx); + } + private: - bool InsertPreparingQuery(const TKqpCompileResult::TConstPtr& compileResult, bool keepInCache) { + bool InsertPreparingQuery(const TKqpCompileResult::TConstPtr& compileResult, bool keepInCache, bool isPerStatement) { YQL_ENSURE(compileResult->Query); auto query = *compileResult->Query; @@ -930,7 +950,7 @@ class TKqpCompileService : public TActorBootstrapped { auto newCompileResult = TKqpCompileResult::Make(CreateGuidAsString(), compileResult->Status, compileResult->Issues, compileResult->MaxReadType, std::move(query), compileResult->Ast); newCompileResult->AllowCache = compileResult->AllowCache; newCompileResult->PreparedQuery = compileResult->PreparedQuery; - return QueryCache.Insert(newCompileResult, TableServiceConfig.GetEnableAstCache()); + return QueryCache.Insert(newCompileResult, TableServiceConfig.GetEnableAstCache(), isPerStatement); } void ProcessQueue(const TActorContext& ctx) { @@ -963,7 +983,7 @@ class TKqpCompileService : public TActorBootstrapped { void StartCompilation(TKqpCompileRequest&& request, const TActorContext& ctx) { auto compileActor = CreateKqpCompileActor(ctx.SelfID, KqpSettings, TableServiceConfig, QueryServiceConfig, MetadataProviderConfig, ModuleResolverState, Counters, request.Uid, request.Query, request.UserToken, FederatedQuerySetup, request.DbCounters, request.UserRequestContext, - request.CompileServiceSpan.GetTraceId(), request.TempTablesState, request.Action, std::move(request.AstResult), CollectDiagnostics); + request.CompileServiceSpan.GetTraceId(), request.TempTablesState, request.Action, std::move(request.QueryAst), CollectDiagnostics, request.CanDevideIntoStatements); auto compileActorId = ctx.ExecutorThread.RegisterActor(compileActor, TMailboxType::HTSwap, AppData(ctx)->UserPoolId); @@ -1040,6 +1060,32 @@ class TKqpCompileService : public TActorBootstrapped { << ", message: " << e.what()); } + void Reply(const TActorId& sender, const TVector astStatements, const TKqpQueryId query, + const TActorContext& ctx, ui64 cookie, NLWTrace::TOrbit orbit, NWilson::TSpan span) + { + LWTRACK(KqpCompileServiceReply, + orbit, + query.UserSid, + {}); + + LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Send ast statements response"); + + auto responseEv = MakeHolder(std::move(query), astStatements, std::move(orbit)); + + if (span) { + span.End(); + } + + ctx.Send(sender, responseEv.Release(), 0, cookie); + } + + void ReplyQueryStatements(const TActorId& sender, const TVector astStatements, + const TKqpQueryId query, const TActorContext& ctx, ui64 cookie, NLWTrace::TOrbit orbit, NWilson::TSpan span) + { + LWTRACK(KqpCompileServiceReplyStatements, orbit); + Reply(sender, astStatements, query, ctx, cookie, std::move(orbit), std::move(span)); + } + private: TTableServiceConfig TableServiceConfig; TQueryServiceConfig QueryServiceConfig; diff --git a/ydb/core/kqp/compile_service/kqp_compile_service.h b/ydb/core/kqp/compile_service/kqp_compile_service.h index 213dc1ecffc2..af37a4596e3d 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_service.h +++ b/ydb/core/kqp/compile_service/kqp_compile_service.h @@ -36,8 +36,9 @@ IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstP NWilson::TTraceId traceId = {}, TKqpTempTablesState::TConstPtr tempTablesState = nullptr, ECompileActorAction compileAction = ECompileActorAction::COMPILE, - TMaybe astResult = {}, - bool collectFullDiagnostics = false); + TMaybe queryAst = {}, + bool collectFullDiagnostics = false, + bool canDevideIntoStatements = false); IActor* CreateKqpCompileRequestActor(const TActorId& owner, const TIntrusiveConstPtr& userToken, const TMaybe& uid, TMaybe&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters, diff --git a/ydb/core/kqp/host/kqp_translate.cpp b/ydb/core/kqp/host/kqp_translate.cpp index a0c97da736b0..2804202040a2 100644 --- a/ydb/core/kqp/host/kqp_translate.cpp +++ b/ydb/core/kqp/host/kqp_translate.cpp @@ -56,7 +56,7 @@ NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type) { NSQLTranslation::TTranslationSettings GetTranslationSettings(NYql::EKikimrQueryType queryType, const TMaybe& usePgParser, bool sqlAutoCommit, const TString& queryText, std::shared_ptr> queryParameters, TMaybe& sqlVersion, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, - NYql::TExprContext& ctx, bool isEnablePgConstsToParams) { + NYql::TExprContext& ctx, bool isEnablePgConstsToParams, bool perStatement) { NSQLTranslation::TTranslationSettings settings{}; if (usePgParser) { @@ -145,6 +145,8 @@ NSQLTranslation::TTranslationSettings GetTranslationSettings(NYql::EKikimrQueryT } } + settings.PerStatement = perStatement; + return settings; } @@ -199,5 +201,55 @@ TQueryAst ParseQuery(NYql::EKikimrQueryType queryType, const TMaybe(std::move(astRes)), sqlVersion, deprecatedSQL); } +TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TMaybe& usePgParser, const TString& queryText, + std::shared_ptr> queryParameters, bool sqlAutoCommit, + TMaybe& sqlVersion, TString cluster, TString kqpTablePathPrefix, + ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, + NYql::TExprContext& ctx, bool isEnablePgConstsToParams, bool isSql, bool perStatement) { + + TVector result; + NYql::TAstParseResult astRes; + if (isSql) { + auto settings = GetTranslationSettings(queryType, usePgParser, sqlAutoCommit, queryText, queryParameters, sqlVersion, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, ctx, isEnablePgConstsToParams, perStatement); + ui16 actualSyntaxVersion = 0; + auto astStatements = NSQLTranslation::SqlToAstStatements(queryText, settings, nullptr, &actualSyntaxVersion); + sqlVersion = actualSyntaxVersion; + for (auto&& ast : astStatements) { + result.push_back({std::make_shared(std::move(ast)), sqlVersion, (actualSyntaxVersion == 0)}); + } + return result; + } else { + sqlVersion = {}; + return {{std::make_shared(NYql::ParseAst(queryText)), sqlVersion, true}}; + } +} + +TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TMaybe& syntax, const TString& queryText, std::shared_ptr> queryParameters, + TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, bool isEnablePgConstsToParams, bool isSql, + bool perStatement) { + TMaybe sqlVersion; + TMaybe usePgParser; + if (syntax) + switch (*syntax) { + case Ydb::Query::Syntax::SYNTAX_YQL_V1: + usePgParser = false; + break; + case Ydb::Query::Syntax::SYNTAX_PG: + usePgParser = true; + break; + default: + break; + } + + NYql::TExprContext ctx; + bool sqlAutoCommit; + if (queryType == NYql::EKikimrQueryType::YqlScript || queryType == NYql::EKikimrQueryType::YqlScriptStreaming) { + sqlAutoCommit = true; + } else { + sqlAutoCommit = false; + } + return SqlToAstStatements(queryType, usePgParser, queryText, queryParameters, sqlAutoCommit, sqlVersion, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, ctx, isEnablePgConstsToParams, isSql, perStatement); +} + } // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/kqp/host/kqp_translate.h b/ydb/core/kqp/host/kqp_translate.h index bf218b8464a0..44c7158bce8f 100644 --- a/ydb/core/kqp/host/kqp_translate.h +++ b/ydb/core/kqp/host/kqp_translate.h @@ -13,7 +13,7 @@ NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type); NSQLTranslation::TTranslationSettings GetTranslationSettings(NYql::EKikimrQueryType queryType, const TMaybe& usePgParser, bool sqlAutoCommit, const TString& queryText, std::shared_ptr> queryParameters, TMaybe& sqlVersion, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, NYql::TExprContext& ctx, - bool isEnablePgConstsToParams); + bool isEnablePgConstsToParams, bool perStatement = false); NYql::TAstParseResult ParseQuery(NYql::EKikimrQueryType queryType, const TMaybe& usePgParser, const TString& queryText, std::shared_ptr> queryParameters, bool isSql, bool sqlAutoCommit, TMaybe& sqlVersion, @@ -24,5 +24,8 @@ TQueryAst ParseQuery(NYql::EKikimrQueryType queryType, const TMaybe> queryParameters, bool isSql, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, bool isEnablePgConstsToParams); +TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TMaybe& syntax, const TString& queryText, std::shared_ptr> queryParameters, + TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, bool isEnablePgConstsToParams, bool isSql, bool perStatement); + } // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index f6fb2beb1e2e..d5cb17ade022 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -162,6 +162,7 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi bool EnableAstCache = false; bool EnablePgConstsToParams = false; ui64 ExtractPredicateRangesLimit = 0; + bool EnableQueriesPerStatement = false; }; } diff --git a/ydb/core/kqp/session_actor/kqp_query_state.cpp b/ydb/core/kqp/session_actor/kqp_query_state.cpp index 9889ac5dee3f..0f7f6e3a3a41 100644 --- a/ydb/core/kqp/session_actor/kqp_query_state.cpp +++ b/ydb/core/kqp/session_actor/kqp_query_state.cpp @@ -123,6 +123,12 @@ bool TKqpQueryState::SaveAndCheckCompileResult(TEvKqp::TEvCompileResponse* ev) { return true; } +bool TKqpQueryState::SaveAndCheckParseResult(TEvKqp::TEvParseResponse* ev) { + Statements = std::move(ev->AstStatements); + Orbit = std::move(ev->Orbit); + return true; +} + std::unique_ptr TKqpQueryState::BuildCompileRequest(std::shared_ptr> cookie) { TMaybe query; TMaybe uid; @@ -134,6 +140,7 @@ std::unique_ptr TKqpQueryState::BuildCompileRequest(s settings.IsPrepareQuery = GetAction() == NKikimrKqp::QUERY_ACTION_PREPARE; bool keepInCache = false; + bool canDevideIntoStatements = !HasTxControl(); switch (GetAction()) { case NKikimrKqp::QUERY_ACTION_EXECUTE: query = TKqpQueryId(Cluster, Database, GetQuery(), settings, GetQueryParameterTypes()); @@ -143,16 +150,19 @@ std::unique_ptr TKqpQueryState::BuildCompileRequest(s case NKikimrKqp::QUERY_ACTION_PREPARE: query = TKqpQueryId(Cluster, Database, GetQuery(), settings, GetQueryParameterTypes()); keepInCache = query->IsSql(); + canDevideIntoStatements = false; break; case NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED: uid = GetPreparedQuery(); keepInCache = GetQueryKeepInCache(); + canDevideIntoStatements = false; break; case NKikimrKqp::QUERY_ACTION_EXPLAIN: query = TKqpQueryId(Cluster, Database, GetQuery(), settings, GetQueryParameterTypes()); keepInCache = false; + canDevideIntoStatements = false; break; default: @@ -164,8 +174,14 @@ std::unique_ptr TKqpQueryState::BuildCompileRequest(s compileDeadline = Min(compileDeadline, QueryDeadlines.CancelAt); } - return std::make_unique(UserToken, uid, - std::move(query), keepInCache, compileDeadline, DbCounters, std::move(cookie), UserRequestContext, std::move(Orbit), TempTablesState, GetCollectDiagnostics()); + TMaybe statementAst; + if (!Statements.empty()) { + statementAst = Statements[CurrentStatementId]; + } + + return std::make_unique(UserToken, uid, std::move(query), keepInCache, + canDevideIntoStatements, compileDeadline, DbCounters, std::move(cookie), UserRequestContext, + std::move(Orbit), TempTablesState, GetCollectDiagnostics(), statementAst); } std::unique_ptr TKqpQueryState::BuildReCompileRequest(std::shared_ptr> cookie) { diff --git a/ydb/core/kqp/session_actor/kqp_query_state.h b/ydb/core/kqp/session_actor/kqp_query_state.h index 7766ddd70116..b52117b9d969 100644 --- a/ydb/core/kqp/session_actor/kqp_query_state.h +++ b/ydb/core/kqp/session_actor/kqp_query_state.h @@ -125,6 +125,9 @@ class TKqpQueryState : public TNonCopyable { NYql::TIssues Issues; + TVector Statements; + size_t CurrentStatementId = 0; + NKikimrKqp::EQueryAction GetAction() const { return RequestEv->GetAction(); } @@ -368,6 +371,42 @@ class TKqpQueryState : public TNonCopyable { return RequestEv->GetTxControl(); } + bool FinishedStatements() { + return CurrentStatementId + 1 >= Statements.size(); + } + + void PrepareCurrentStatement() { + QueryData = {}; + PreparedQuery = {}; + CompileResult = {}; + TxCtx = {}; + CurrentTx = 0; + TableVersions = {}; + MaxReadType = ETableReadType::Other; + Commit = false; + Commited = false; + TopicOperations = {}; + ReplayMessage = {}; + } + + void PrepareNextStatement() { + CurrentStatementId++; + PrepareCurrentStatement(); + } + + void PrepareStatementTransaction(NKqpProto::TKqpPhyTx_EType txType) { + if (!HasTxControl()) { + switch (txType) { + case NKqpProto::TKqpPhyTx::TYPE_SCHEME: + TxCtx->EffectiveIsolationLevel = NKikimrKqp::ISOLATION_LEVEL_UNDEFINED; + break; + default: + Commit = true; + TxCtx->EffectiveIsolationLevel = NKikimrKqp::ISOLATION_LEVEL_SERIALIZABLE; + } + } + } + // validate the compiled query response and ensure that all table versions are not // changed since the last compilation. bool EnsureTableVersions(const TEvTxProxySchemeCache::TEvNavigateKeySetResult& response); @@ -376,6 +415,7 @@ class TKqpQueryState : public TNonCopyable { std::unique_ptr BuildNavigateKeySet(); // same the context of the compiled query to the query state. bool SaveAndCheckCompileResult(TEvKqp::TEvCompileResponse* ev); + bool SaveAndCheckParseResult(TEvKqp::TEvParseResponse* ev); // build the compilation request. std::unique_ptr BuildCompileRequest(std::shared_ptr> cookie); // TODO(gvit): get rid of code duplication in these requests, diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index 13e221e164de..a69555932e86 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -78,7 +79,7 @@ std::optional TryDecodeYdbSessionId(const TString& sessionId) { #define LOG_W(msg) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) #define LOG_N(msg) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) #define LOG_I(msg) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) -#define LOG_D(msg) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) +#define LOG_D(msg) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) #define LOG_T(msg) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) void FillColumnsMeta(const NKqpProto::TKqpPhyQuery& phyQuery, NKikimrKqp::TQueryResponse& resp) { @@ -512,6 +513,19 @@ class TKqpSessionActor : public TActorBootstrapped { OnSuccessCompileRequest(); } + void Handle(TEvKqp::TEvParseResponse::TPtr& ev) { + QueryState->SaveAndCheckParseResult(ev->Get()); + CompileStatement(); + } + + void CompileStatement() { + auto request = QueryState->BuildCompileRequest(CompilationCookie); + LOG_D("Sending CompileQuery request"); + + Send(MakeKqpCompileServiceID(SelfId().NodeId()), request.release(), 0, QueryState->QueryId, + QueryState->KqpSessionSpan.GetTraceId()); + } + void OnSuccessCompileRequest() { if (QueryState->GetAction() == NKikimrKqp::QUERY_ACTION_PREPARE || QueryState->GetAction() == NKikimrKqp::QUERY_ACTION_EXPLAIN) @@ -680,6 +694,9 @@ class TKqpSessionActor : public TActorBootstrapped { AppData()->TimeProvider, AppData()->RandomProvider, Config->EnableKqpImmediateEffects); QueryState->QueryData = std::make_shared(QueryState->TxCtx->TxAlloc); QueryState->TxCtx->EffectiveIsolationLevel = NKikimrKqp::ISOLATION_LEVEL_UNDEFINED; + Ydb::Table::TransactionSettings settings; + settings.mutable_serializable_read_write(); + BeginTx(settings); } return true; @@ -949,6 +966,7 @@ class TKqpSessionActor : public TActorBootstrapped { bool ExecutePhyTx(const TKqpPhyTxHolder::TConstPtr& tx, bool commit) { if (tx) { + QueryState->PrepareStatementTransaction(tx->GetType()); switch (tx->GetType()) { case NKqpProto::TKqpPhyTx::TYPE_SCHEME: YQL_ENSURE(tx->StagesSize() == 0); @@ -1603,7 +1621,16 @@ class TKqpSessionActor : public TActorBootstrapped { QueryResponse = std::move(resEv); - Cleanup(); + ProcessNextStatement(); + } + + void ProcessNextStatement() { + if (QueryState->FinishedStatements()) { + Cleanup(); + return; + } + QueryState->PrepareNextStatement(); + CompileStatement(); } void ReplyQueryCompileError() { @@ -2085,6 +2112,7 @@ class TKqpSessionActor : public TActorBootstrapped { // forgotten messages from previous aborted request hFunc(TEvKqp::TEvCompileResponse, Handle); + hFunc(TEvKqp::TEvParseResponse, Handle); hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); hFunc(TEvents::TEvUndelivered, HandleNoop); diff --git a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp index 5f01ec1ef5db..4f3cc59613b5 100644 --- a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp +++ b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp @@ -3680,6 +3680,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { } appConfig.MutableTableServiceConfig()->SetEnablePgConstsToParams(true); + appConfig.MutableTableServiceConfig()->SetEnableQueriesPerStatement(true); auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetAppConfig(appConfig) @@ -3716,6 +3717,32 @@ Y_UNIT_TEST_SUITE(KqpPg) { } } + { + // Check NoTx + { + const auto query = Q_(R"( + CREATE TABLE PgTable3 ( + key int4 PRIMARY KEY, + value text + ); + SELECT * FROM PgTable3 WHERE key = 3; + )"); + auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx(), settings).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { + const auto query = Q_(R"( + SELECT * FROM PgTable3 WHERE key = 4; + SELECT * FROM PgTable3 WHERE key = 5; + )"); + auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx(), settings).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), true); + } + } + { // Check values without table { diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index f7a2639a5968..404423bfc040 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -1764,6 +1764,130 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT(!service->IsUnsafeToShutdown()); } } + + Y_UNIT_TEST(Ddl_Dml) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnablePreparedDdl(true); + appConfig.MutableTableServiceConfig()->SetEnableAstCache(true); + appConfig.MutableTableServiceConfig()->SetEnableQueriesPerStatement(true); + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetAppConfig(appConfig) + .SetKqpSettings({setting}); + + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetQueryClient(); + + { + // Base test with ddl and adm statements + auto result = db.ExecuteQuery(R"( + DECLARE $name AS Text; + $a = (SELECT * FROM TestDdl1); + CREATE TABLE TestDdl1 ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + UPSERT INTO TestDdl1 (Key, Value) VALUES (1, "One"); + CREATE TABLE TestDdl2 ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + UPSERT INTO TestDdl1 (Key, Value) VALUES (2, "Two"); + SELECT * FROM $a; + )", TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 1); + CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", FormatResultSetYson(result.GetResultSet(0))); + + result = db.ExecuteQuery(R"( + UPSERT INTO TestDdl1 (Key, Value) VALUES (3, "Three"); + )", TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + result = db.ExecuteQuery(R"( + SELECT * FROM TestDdl1; + )", TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 1); + CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]]])", FormatResultSetYson(result.GetResultSet(0))); + + result = db.ExecuteQuery(R"( + CREATE TABLE TestDdl1 ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )", TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Check failed: path: '/Root/TestDdl1', error: path exist")); + + result = db.ExecuteQuery(R"( + CREATE TABLE TestDdl2 ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )", TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Check failed: path: '/Root/TestDdl2', error: path exist")); + } + + { + // Test with query with error + auto result = db.ExecuteQuery(R"( + UPSERT INTO TestDdl2 (Key, Value) VALUES (1, "One"); + CREATE TABLE TestDdl3 ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + UPSERT INTO TestDdl2 (Key, Value) VALUES (2, "Two"); + CREATE TABLE TestDdl2 ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + CREATE TABLE TestDdl4 ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + UPSERT INTO TestDdl1 (Key, Value) VALUES (3, "Thee"); + )", TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + + result = db.ExecuteQuery(R"( + SELECT * FROM TestDdl2; + )", TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 1); + CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", FormatResultSetYson(result.GetResultSet(0))); + } + + { + // Test with several select + auto result = db.ExecuteQuery(R"( + $a = (SELECT * FROM TestDdl1); + SELECT * FROM $a; + UPSERT INTO TestDdl1 (Key, Value) VALUES (4, "Four"); + SELECT * FROM $a; + CREATE TABLE TestDdl4 ( + Key Uint64, + Val Uint64, + PRIMARY KEY (Key) + ); + UPSERT INTO TestDdl4 (Key, Val) VALUES (1, 1); + )", TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + // NEED TO FIX IT: RESULT MUST BE DEVIDED INTO 2 SETS + UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 1); + CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]];[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]];[[4u];["Four"]]])", FormatResultSetYson(result.GetResultSet(0))); + //CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]]])", FormatResultSetYson(result.GetResultSet(0))); + //CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]][[4u];["Four"]]])", FormatResultSetYson(result.GetResultSet(1))); + } + } } } // namespace NKqp diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 4e405b9f3c93..ae6e590c731c 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -105,7 +105,7 @@ message TFeatureFlags { optional bool EnableSeparationComputeActorsFromRead = 90 [default = false]; optional bool EnablePQConfigTransactionsAtSchemeShard = 91 [default = false]; optional bool EnableScriptExecutionOperations = 92 [default = false]; - optional bool EnableImplicitQueryParameterTypes = 93 [default = true]; + optional bool EnableImplicitQueryParameterTypes = 93 [default = false]; optional bool EnableForceImmediateEffectsExecution = 94 [default = false]; optional bool EnableTopicSplitMerge = 95 [default = false]; optional bool EnableChangefeedDynamoDBStreamsFormat = 96 [default = true]; diff --git a/ydb/core/protos/table_service_config.proto b/ydb/core/protos/table_service_config.proto index 9f00e30fbf7d..1a4cf4d0c1b5 100644 --- a/ydb/core/protos/table_service_config.proto +++ b/ydb/core/protos/table_service_config.proto @@ -272,5 +272,8 @@ message TTableServiceConfig { optional bool EnablePgConstsToParams = 53 [default = false]; optional uint64 ExtractPredicateRangesLimit = 54 [default = 10000]; + optional bool EnableOlapSink = 55 [default = false]; + + optional bool EnableQueriesPerStatement = 56 [default = false]; }; diff --git a/ydb/library/yql/parser/pg_wrapper/interface/parser.h b/ydb/library/yql/parser/pg_wrapper/interface/parser.h index d481820e7ef1..e5a019213c7e 100644 --- a/ydb/library/yql/parser/pg_wrapper/interface/parser.h +++ b/ydb/library/yql/parser/pg_wrapper/interface/parser.h @@ -11,5 +11,6 @@ struct TTranslationSettings; namespace NSQLTranslationPG { NYql::TAstParseResult PGToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings); +TVector PGToYqlStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings); } // NSQLTranslationPG diff --git a/ydb/library/yql/parser/pg_wrapper/interface/raw_parser.h b/ydb/library/yql/parser/pg_wrapper/interface/raw_parser.h index dbff24024222..639fee9960e6 100644 --- a/ydb/library/yql/parser/pg_wrapper/interface/raw_parser.h +++ b/ydb/library/yql/parser/pg_wrapper/interface/raw_parser.h @@ -19,4 +19,6 @@ TString PrintPGTree(const List* raw); void PGParse(const TString& input, IPGParseEvents& events); +List* PGGetStatements(const TString& input); + } diff --git a/ydb/library/yql/parser/pg_wrapper/parser.cpp b/ydb/library/yql/parser/pg_wrapper/parser.cpp index 73c82fe2f9e4..17c7990eefa4 100644 --- a/ydb/library/yql/parser/pg_wrapper/parser.cpp +++ b/ydb/library/yql/parser/pg_wrapper/parser.cpp @@ -216,6 +216,35 @@ void PGParse(const TString& input, IPGParseEvents& events) { } } +List* PGGetStatements(const TString& input) { + pg_thread_init(); + + PgQueryInternalParsetreeAndError parsetree_and_error; + + TArenaMemoryContext arena; + auto prevErrorContext = ErrorContext; + ErrorContext = CurrentMemoryContext; + + Y_DEFER { + ErrorContext = prevErrorContext; + }; + + parsetree_and_error = pg_query_raw_parse(input.c_str()); + Y_DEFER { + if (parsetree_and_error.error) { + pg_query_free_error(parsetree_and_error.error); + } + + free(parsetree_and_error.stderr_buffer); + }; + + if (parsetree_and_error.error) { + return nullptr; + } else { + return parsetree_and_error.tree; + } +} + TString PrintPGTree(const List* raw) { auto str = nodeToString(raw); Y_DEFER { diff --git a/ydb/library/yql/sql/pg/pg_sql.cpp b/ydb/library/yql/sql/pg/pg_sql.cpp index 1c05938c6652..22a55bba11f6 100644 --- a/ydb/library/yql/sql/pg/pg_sql.cpp +++ b/ydb/library/yql/sql/pg/pg_sql.cpp @@ -152,7 +152,7 @@ std::shared_ptr ListMake1(void* cell) { #define CAST_NODE(nodeType, nodeptr) CastNode(nodeptr, T_##nodeType) #define CAST_NODE_EXT(nodeType, tag, nodeptr) CastNode(nodeptr, tag) -#define LIST_CAST_NTH(nodeType, list, index) CAST_NODE(nodeType, list_nth(list, i)) +#define LIST_CAST_NTH(nodeType, list, index) CAST_NODE(nodeType, list_nth(list, index)) #define LIST_CAST_EXT_NTH(nodeType, tag, list, index) CAST_NODE_EXT(nodeType, tag, list_nth(list, i)) const Node* ListNodeNth(const List* list, int index) { @@ -327,11 +327,23 @@ class TConverter : public IPGParseEvents { } } + void OnResult(const List* raw, int StatementId) { + if (!Settings.PerStatement) { + return OnResult(raw); + } + + AstParseResult.Pool = std::make_unique(4096); + AstParseResult.Root = ParseResult(raw, StatementId); + if (!AutoParamValues.empty()) { + AstParseResult.PgAutoParamValues = std::move(AutoParamValues); + } + } + void OnError(const TIssue& issue) { AstParseResult.Issues.AddIssue(issue); } - TAstNode* ParseResult(const List* raw) { + TAstNode* ParseResult(const List* raw, const TMaybe& statementId = Nothing()) { auto configSource = L(A("DataSource"), QA(TString(NYql::ConfigProviderName))); Statements.push_back(L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, QA("OrderedColumns")))); @@ -343,10 +355,16 @@ class TConverter : public IPGParseEvents { ui32 dqEnginePgmPos = Statements.size(); Statements.push_back(configSource); - for (int i = 0; i < ListLength(raw); ++i) { - if (!ParseRawStmt(LIST_CAST_NTH(RawStmt, raw, i))) { + if (statementId) { + if (!ParseRawStmt(LIST_CAST_NTH(RawStmt, raw, *statementId))) { return nullptr; } + } else { + for (int i = 0; i < ListLength(raw); ++i) { + if (!ParseRawStmt(LIST_CAST_NTH(RawStmt, raw, i))) { + return nullptr; + } + } } if (!Views.empty()) { @@ -4460,7 +4478,24 @@ NYql::TAstParseResult PGToYql(const TString& query, const NSQLTranslation::TTran NYql::TAstParseResult result; TConverter converter(result, settings, query); NYql::PGParse(query, converter); - return result; + return std::move(result); +} + +TVector PGToYqlStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings) { + TVector results; + size_t currentStatementId = 0; + bool finished; + List* raw = NYql::PGGetStatements(query); + if (!raw) { + return {}; + } + for (int statementId = 0; statementId < ListLength(raw); ++statementId) { + NYql::TAstParseResult result; + TConverter converter(result, settings, query); + converter.OnResult(raw, statementId); + results.push_back(std::move(result)); + } + return results; } } // NSQLTranslationPG diff --git a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp index 9811c49b5f05..5a2c605e9296 100644 --- a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp +++ b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp @@ -13,6 +13,12 @@ NYql::TAstParseResult PGToYql(const TString& query, const NSQLTranslation::TTran return result; } +TVector PGToYqlStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings) { + Y_UNUSED(query); + Y_UNUSED(settings); + return {}; +} + } // NSQLTranslationPG namespace NYql { diff --git a/ydb/library/yql/sql/settings/translation_settings.h b/ydb/library/yql/sql/settings/translation_settings.h index 8e00ace30d96..4667d70d6cd6 100644 --- a/ydb/library/yql/sql/settings/translation_settings.h +++ b/ydb/library/yql/sql/settings/translation_settings.h @@ -107,6 +107,7 @@ namespace NSQLTranslation { bool AssumeYdbOnClusterWithSlash; TString DynamicClusterProvider; TString FileAliasPrefix; + bool PerStatement = false; TVector PgParameterTypeOids; bool AutoParametrizeEnabled = false; diff --git a/ydb/library/yql/sql/sql.cpp b/ydb/library/yql/sql/sql.cpp index 8d5371894f5f..c5746485c3e4 100644 --- a/ydb/library/yql/sql/sql.cpp +++ b/ydb/library/yql/sql/sql.cpp @@ -78,6 +78,11 @@ namespace NSQLTranslation { *actualSyntaxVersion = parsedSettings.SyntaxVersion; } + google::protobuf::Arena arena; + if (!parsedSettings.Arena) { + parsedSettings.Arena = &arena; + } + switch (parsedSettings.SyntaxVersion) { case 0: if (settings.V0ForceDisable || settings.V0Behavior == EV0Behavior::Disable) { @@ -92,9 +97,9 @@ namespace NSQLTranslation { return nullptr; } - return NSQLTranslationV0::SqlAST(query, queryName, issues, maxErrors, settings.Arena); + return NSQLTranslationV0::SqlAST(query, queryName, issues, maxErrors, parsedSettings.Arena); case 1: - return NSQLTranslationV1::SqlAST(query, queryName, issues, maxErrors, parsedSettings.AnsiLexer, settings.Arena); + return NSQLTranslationV1::SqlAST(query, queryName, issues, maxErrors, parsedSettings.AnsiLexer, parsedSettings.Arena); default: issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, TStringBuilder() << "Unknown SQL syntax version: " << parsedSettings.SyntaxVersion)); @@ -163,4 +168,47 @@ namespace NSQLTranslation { } } + TVector SqlToAstStatements(const TString& query, const TTranslationSettings& settings, + NYql::TWarningRules* warningRules, ui16* actualSyntaxVersion) + { + TVector result; + NYql::TIssues issues; + TTranslationSettings parsedSettings(settings); + google::protobuf::Arena arena; + if (!parsedSettings.Arena) { + parsedSettings.Arena = &arena; + } + + if (!ParseTranslationSettings(query, parsedSettings, issues)) { + return {}; + } + + if (actualSyntaxVersion) { + *actualSyntaxVersion = parsedSettings.SyntaxVersion; + } + + if (!parsedSettings.DeclaredNamedExprs.empty() && !parsedSettings.PgParser && parsedSettings.SyntaxVersion != 1) { + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, + "Externally declared named expressions not supported in V0 syntax")); + return {}; + } + + if (parsedSettings.PgParser) { + return NSQLTranslationPG::PGToYqlStatements(query, parsedSettings); + } + + switch (parsedSettings.SyntaxVersion) { + case 0: + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, + "V0 syntax is disabled")); + return {}; + case 1: + return NSQLTranslationV1::SqlToAstStatements(query, parsedSettings, warningRules); + default: + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, + TStringBuilder() << "Unknown SQL syntax version: " << parsedSettings.SyntaxVersion)); + return {}; + } + } + } // namespace NSQLTranslation diff --git a/ydb/library/yql/sql/sql.h b/ydb/library/yql/sql/sql.h index db699ebece13..d775555e3b26 100644 --- a/ydb/library/yql/sql/sql.h +++ b/ydb/library/yql/sql/sql.h @@ -21,5 +21,7 @@ namespace NSQLTranslation { const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr); ILexer::TPtr SqlLexer(const TString& query, NYql::TIssues& issues, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr); NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings); + TVector SqlToAstStatements(const TString& query, const TTranslationSettings& settings, + NYql::TWarningRules* warningRules = nullptr, ui16* actualSyntaxVersion = nullptr); } // namespace NSQLTranslationV0 diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index c9238d3b5c1b..3cf44e61515b 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -27,6 +27,20 @@ TAstNode* SqlASTToYql(const google::protobuf::Message& protoAst, TContext& ctx) return nullptr; } +TAstNode* SqlASTsToYqls(const std::vector<::NSQLv1Generated::TRule_sql_stmt_core>& ast, TContext& ctx) { + TSqlQuery query(ctx, ctx.Settings.Mode, true); + TNodePtr node(query.Build(ast)); + try { + if (node && node->Init(ctx, nullptr)) { + return node->Translate(ctx); + } + } catch (const NProtoAST::TTooManyErrors&) { + // do not add error issue, no room for it + } + + return nullptr; +} + void SqlASTToYqlImpl(NYql::TAstParseResult& res, const google::protobuf::Message& protoAst, TContext& ctx) { YQL_ENSURE(!ctx.Issues.Size()); @@ -45,6 +59,22 @@ void SqlASTToYqlImpl(NYql::TAstParseResult& res, const google::protobuf::Message } } +void SqlASTsToYqlsImpl(NYql::TAstParseResult& res, const std::vector<::NSQLv1Generated::TRule_sql_stmt_core>& ast, TContext& ctx) { + res.Root = SqlASTsToYqls(ast, ctx); + res.Pool = std::move(ctx.Pool); + if (!res.Root) { + if (ctx.Issues.Size()) { + ctx.IncrementMonCounter("sql_errors", "AstToYqlError"); + } else { + ctx.IncrementMonCounter("sql_errors", "AstToYqlSilentError"); + ctx.Error() << "Error occurred on parse SQL query, but no error is collected" << + ", please send this request over bug report into YQL interface or write on yql@ maillist"; + } + } else { + ctx.WarnUnusedHints(); + } +} + NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const NSQLTranslation::TSQLHints& hints, const NSQLTranslation::TTranslationSettings& settings) @@ -84,4 +114,72 @@ NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTra return res; } +bool NeedUseForAllStatements(const TRule_sql_stmt_core::AltCase& subquery) { + return subquery == TRule_sql_stmt_core::kAltSqlStmtCore1 || // pragma + subquery == TRule_sql_stmt_core::kAltSqlStmtCore3 || // named nodes + subquery == TRule_sql_stmt_core::kAltSqlStmtCore6 || // use + subquery == TRule_sql_stmt_core::kAltSqlStmtCore12 || // declare + subquery == TRule_sql_stmt_core::kAltSqlStmtCore13 || // import + subquery == TRule_sql_stmt_core::kAltSqlStmtCore14 || // export + subquery == TRule_sql_stmt_core::kAltSqlStmtCore17; // define action or subquery +} + +TVector SqlToAstStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules) +{ + TVector res; + const TString queryName = "query"; + TIssues issues; + + NSQLTranslation::TSQLHints hints; + auto lexer = MakeLexer(settings.AnsiLexer); + YQL_ENSURE(lexer); + if (!CollectSqlHints(*lexer, query, queryName, settings.File, hints, issues, settings.MaxErrors)) { + return res; + } + + TContext ctx(settings, hints, issues); + NSQLTranslation::TErrorCollectorOverIssues collector(issues, settings.MaxErrors, settings.File); + + google::protobuf::Message* astProto(SqlAST(query, queryName, collector, settings.AnsiLexer, settings.Arena)); + if (astProto) { + auto ast = static_cast(*astProto); + const auto& query = ast.GetRule_sql_query(); + if (!settings.PerStatement) { + res.emplace_back(); + SqlASTToYqlImpl(res.back(), *astProto, ctx); + return res; + } + if (query.Alt_case() == NSQLv1Generated::TRule_sql_query::kAltSqlQuery1) { + std::vector<::NSQLv1Generated::TRule_sql_stmt_core> commonStates; + std::vector<::NSQLv1Generated::TRule_sql_stmt_core> result; + const auto& statements = query.GetAlt_sql_query1().GetRule_sql_stmt_list1(); + if (NeedUseForAllStatements(statements.GetRule_sql_stmt2().GetRule_sql_stmt_core2().Alt_case())) { + commonStates.push_back(statements.GetRule_sql_stmt2().GetRule_sql_stmt_core2()); + } else { + TContext ctx(settings, hints, issues); + res.emplace_back(); + SqlASTsToYqlsImpl(res.back(), {statements.GetRule_sql_stmt2().GetRule_sql_stmt_core2()}, ctx); + } + for (auto block: statements.GetBlock3()) { + if (NeedUseForAllStatements(block.GetRule_sql_stmt2().GetRule_sql_stmt_core2().Alt_case())) { + commonStates.push_back(block.GetRule_sql_stmt2().GetRule_sql_stmt_core2()); + continue; + } + TContext ctx(settings, hints, issues); + res.emplace_back(); + result = commonStates; + result.push_back(block.GetRule_sql_stmt2().GetRule_sql_stmt_core2()); + SqlASTsToYqlsImpl(res.back(), result, ctx); + } + } + } else { + ctx.IncrementMonCounter("sql_errors", "AstError"); + } + if (warningRules) { + *warningRules = ctx.WarningPolicy.GetRules(); + ctx.WarningPolicy.Clear(); + } + return res; +} + } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/sql.h b/ydb/library/yql/sql/v1/sql.h index 82e5c97a6978..8283b9ec6f7b 100644 --- a/ydb/library/yql/sql/v1/sql.h +++ b/ydb/library/yql/sql/v1/sql.h @@ -17,5 +17,6 @@ namespace NSQLTranslationV1 { NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules = nullptr); NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const NSQLTranslation::TSQLHints& hints, const NSQLTranslation::TTranslationSettings& settings); + TVector SqlToAstStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules); } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/sql_query.cpp b/ydb/library/yql/sql/v1/sql_query.cpp index 38f141e0cd3d..b50f059a399c 100644 --- a/ydb/library/yql/sql/v1/sql_query.cpp +++ b/ydb/library/yql/sql/v1/sql_query.cpp @@ -2660,6 +2660,76 @@ TNodePtr TSqlQuery::Build(const TSQLv1ParserAST& ast) { WarnUnusedNodes(); return result; } + +TNodePtr TSqlQuery::Build(const std::vector<::NSQLv1Generated::TRule_sql_stmt_core>& statements) { + if (Mode == NSQLTranslation::ESqlMode::QUERY) { + // inject externally declared named expressions + for (auto [name, type] : Ctx.Settings.DeclaredNamedExprs) { + if (name.empty()) { + Error() << "Empty names for externally declared expressions are not allowed"; + return nullptr; + } + TString varName = "$" + name; + if (IsAnonymousName(varName)) { + Error() << "Externally declared name '" << name << "' is anonymous"; + return nullptr; + } + + auto parsed = ParseType(type, *Ctx.Pool, Ctx.Issues, Ctx.Pos()); + if (!parsed) { + Error() << "Failed to parse type for externally declared name '" << name << "'"; + return nullptr; + } + + TNodePtr typeNode = BuildBuiltinFunc(Ctx, Ctx.Pos(), "ParseType", { BuildLiteralRawString(Ctx.Pos(), type) }); + PushNamedAtom(Ctx.Pos(), varName); + // no duplicates are possible at this stage + bool isWeak = true; + Ctx.DeclareVariable(varName, typeNode, isWeak); + // avoid 'Symbol is not used' warning for externally declared expression + YQL_ENSURE(GetNamedNode(varName)); + } + } + + TVector blocks; + Ctx.PushCurrentBlocks(&blocks); + Y_DEFER { + Ctx.PopCurrentBlocks(); + }; + for (const auto& statement : statements) { + if (!Statement(blocks, statement)) { + return nullptr; + } + } + + ui32 topLevelSelects = 0; + bool hasTailOps = false; + for (auto& block : blocks) { + if (block->SubqueryAlias()) { + continue; + } + + if (block->HasSelectResult()) { + ++topLevelSelects; + } else if (topLevelSelects) { + hasTailOps = true; + } + } + + if ((Mode == NSQLTranslation::ESqlMode::SUBQUERY || Mode == NSQLTranslation::ESqlMode::LIMITED_VIEW) && (topLevelSelects != 1 || hasTailOps)) { + Error() << "Strictly one select/process/reduce statement is expected at the end of " + << (Mode == NSQLTranslation::ESqlMode::LIMITED_VIEW ? "view" : "subquery"); + return nullptr; + } + + if (!Ctx.PragmaAutoCommit && Ctx.Settings.EndOfQueryCommit && IsQueryMode(Mode)) { + AddStatementToBlocks(blocks, BuildCommitClusters(Ctx.Pos())); + } + + auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped); + WarnUnusedNodes(); + return result; +} namespace { static bool BuildColumnFeatures(std::map& result, const TRule_column_schema& columnSchema, const NYql::TPosition& pos, TSqlTranslation& translation) { diff --git a/ydb/library/yql/sql/v1/sql_query.h b/ydb/library/yql/sql/v1/sql_query.h index e97ebd56de2c..97301f96ab8e 100644 --- a/ydb/library/yql/sql/v1/sql_query.h +++ b/ydb/library/yql/sql/v1/sql_query.h @@ -18,6 +18,7 @@ class TSqlQuery: public TSqlTranslation { } TNodePtr Build(const TSQLv1ParserAST& ast); + TNodePtr Build(const std::vector<::NSQLv1Generated::TRule_sql_stmt_core>& ast); bool Statement(TVector& blocks, const TRule_sql_stmt_core& core); private: diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index d640235bf2ed..e98a1b74e2ef 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -2721,6 +2721,17 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT_NO_DIFF(Err2Str(res), "
:1:13: Error: Unexpected character : syntax error...\n\n"); } + Y_UNIT_TEST(AAA) { + TTranslationSettings settings; + settings.PerStatement = true; + auto res = SqlToAstStatements("$a = (SELECT * FROM TestDdl1);\nSELECT * FROM $a;\nUPSERT INTO TestDdl1 (Key, Value) VALUES (3, \"Three\");", settings); + for (auto& i : res) { + if (!i.Root) { + Cerr << Err2Str(i); + } + } + } + Y_UNIT_TEST(InvalidDoubleAtStringWhichWasAcceptedEarlier) { NYql::TAstParseResult res = SqlToYql("SELECT @@foo@@ @ @@bar@@"); UNIT_ASSERT(!res.Root); From 1cf767aa76171e859dfa1f6a89b9583f849aaee6 Mon Sep 17 00:00:00 2001 From: Polina Volosnikova Date: Thu, 1 Feb 2024 12:02:36 +0000 Subject: [PATCH 2/5] fix mistakes in word divide --- ydb/core/kqp/common/compilation/events.h | 6 ++-- .../kqp/compile_service/kqp_compile_actor.cpp | 18 ++++++------ .../compile_service/kqp_compile_service.cpp | 28 +++++++++---------- .../kqp/compile_service/kqp_compile_service.h | 2 +- ydb/core/kqp/host/kqp_translate.cpp | 12 ++++---- ydb/core/kqp/host/kqp_translate.h | 4 +-- ydb/core/kqp/provider/yql_kikimr_settings.h | 2 +- .../kqp/session_actor/kqp_query_state.cpp | 10 +++---- ydb/core/kqp/ut/pg/kqp_pg_ut.cpp | 2 +- ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp | 7 +++-- ydb/core/protos/table_service_config.proto | 2 +- ydb/library/yql/sql/pg/pg_sql.cpp | 4 +-- .../yql/sql/settings/translation_settings.h | 2 +- ydb/library/yql/sql/v1/sql.cpp | 2 +- ydb/library/yql/sql/v1/sql_ut.cpp | 11 -------- 15 files changed, 50 insertions(+), 62 deletions(-) diff --git a/ydb/core/kqp/common/compilation/events.h b/ydb/core/kqp/common/compilation/events.h index a1a76be65b25..d5d632bfb8e0 100644 --- a/ydb/core/kqp/common/compilation/events.h +++ b/ydb/core/kqp/common/compilation/events.h @@ -14,7 +14,7 @@ namespace NKikimr::NKqp::NPrivateEvents { struct TEvCompileRequest: public TEventLocal { TEvCompileRequest(const TIntrusiveConstPtr& userToken, const TMaybe& uid, - TMaybe&& query, bool keepInCache, bool canDevideIntoStatements, TInstant deadline, + TMaybe&& query, bool keepInCache, bool canDivideIntoStatements, TInstant deadline, TKqpDbCountersPtr dbCounters, std::shared_ptr> intrestedInResult, const TIntrusivePtr& userRequestContext, NLWTrace::TOrbit orbit = {}, TKqpTempTablesState::TConstPtr tempTablesState = nullptr, bool collectDiagnostics = false, TMaybe queryAst = Nothing()) @@ -22,7 +22,7 @@ struct TEvCompileRequest: public TEventLocal Uid; TMaybe Query; bool KeepInCache = false; - bool CanDevideIntoStatements = false; + bool CanDivideIntoStatements = false; // it is allowed for local event to use absolute time (TInstant) instead of time interval (TDuration) TInstant Deadline; TKqpDbCountersPtr DbCounters; diff --git a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp index f515c9117e48..aea9765b12d4 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp @@ -52,7 +52,7 @@ class TKqpCompileActor : public TActorBootstrapped { TKqpDbCountersPtr dbCounters, std::optional federatedQuerySetup, const TIntrusivePtr& userRequestContext, NWilson::TTraceId traceId, TKqpTempTablesState::TConstPtr tempTablesState, bool collectFullDiagnostics, - bool canDevideIntoStatements, ECompileActorAction compileAction, TMaybe queryAst) + bool canDivideIntoStatements, ECompileActorAction compileAction, TMaybe queryAst) : Owner(owner) , ModuleResolverState(moduleResolverState) , Counters(counters) @@ -70,7 +70,7 @@ class TKqpCompileActor : public TActorBootstrapped { , CompileActorSpan(TWilsonKqp::CompileActor, std::move(traceId), "CompileActor") , TempTablesState(std::move(tempTablesState)) , CollectFullDiagnostics(collectFullDiagnostics) - , CanDevideIntoStatements(canDevideIntoStatements) + , CanDivideIntoStatements(canDivideIntoStatements) , CompileAction(compileAction) , QueryAst(std::move(queryAst)) { @@ -130,15 +130,15 @@ class TKqpCompileActor : public TActorBootstrapped { private: TVector GetAstStatements(const TActorContext &ctx) { - TString cluster = QueryId.Cluster; + TString cluster = QueryId.Cluster; TString kqpTablePathPrefix = Config->_KqpTablePathPrefix.Get().GetRef(); ui16 kqpYqlSyntaxVersion = Config->_KqpYqlSyntaxVersion.Get().GetRef(); NSQLTranslation::EBindingsMode bindingsMode = Config->BindingsMode; bool isEnableExternalDataSources = AppData(ctx)->FeatureFlags.GetEnableExternalDataSources(); bool isEnablePgConstsToParams = Config->EnablePgConstsToParams; - bool perStatement = Config->EnableQueriesPerStatement && CanDevideIntoStatements; + bool perStatementExecution = Config->EnablePerStatementQueryExecution && CanDivideIntoStatements; - return SqlToAstStatements(ConvertType(QueryId.Settings.QueryType), QueryId.Settings.Syntax, QueryId.Text, QueryId.QueryParameterTypes, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, isEnablePgConstsToParams, QueryId.IsSql(), perStatement); + return SqlToAstStatements(ConvertType(QueryId.Settings.QueryType), QueryId.Settings.Syntax, QueryId.Text, QueryId.QueryParameterTypes, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, isEnablePgConstsToParams, QueryId.IsSql(), perStatementExecution); } void StartParsing(const TActorContext &ctx) { @@ -501,7 +501,7 @@ class TKqpCompileActor : public TActorBootstrapped { TKqpTempTablesState::TConstPtr TempTablesState; bool CollectFullDiagnostics; - bool CanDevideIntoStatements; + const bool CanDivideIntoStatements; ECompileActorAction CompileAction; TMaybe QueryAst; }; @@ -532,7 +532,7 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf kqpConfig.IndexAutoChooserMode = serviceConfig.GetIndexAutoChooseMode(); kqpConfig.EnablePgConstsToParams = serviceConfig.GetEnablePgConstsToParams() && serviceConfig.GetEnableAstCache(); kqpConfig.ExtractPredicateRangesLimit = serviceConfig.GetExtractPredicateRangesLimit(); - kqpConfig.EnableQueriesPerStatement = serviceConfig.GetEnableQueriesPerStatement(); + kqpConfig.EnablePerStatementQueryExecution = serviceConfig.GetEnablePerStatementQueryExecution(); if (const auto limit = serviceConfig.GetResourceManager().GetMkqlHeavyProgramMemoryLimit()) { kqpConfig._KqpYqlCombinerMemoryLimit = std::max(1_GB, limit - (limit >> 2U)); @@ -549,14 +549,14 @@ IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstP TKqpDbCountersPtr dbCounters, const TIntrusivePtr& userRequestContext, NWilson::TTraceId traceId, TKqpTempTablesState::TConstPtr tempTablesState, ECompileActorAction compileAction, TMaybe queryAst, bool collectFullDiagnostics, - bool canDevideIntoStatements) + bool canDivideIntoStatements) { return new TKqpCompileActor(owner, kqpSettings, tableServiceConfig, queryServiceConfig, metadataProviderConfig, moduleResolverState, counters, uid, query, userToken, dbCounters, federatedQuerySetup, userRequestContext, std::move(traceId), std::move(tempTablesState), collectFullDiagnostics, - canDevideIntoStatements, compileAction, std::move(queryAst)); + canDivideIntoStatements, compileAction, std::move(queryAst)); } } // namespace NKqp diff --git a/ydb/core/kqp/compile_service/kqp_compile_service.cpp b/ydb/core/kqp/compile_service/kqp_compile_service.cpp index 43db6a69b6b7..45ed1c7bd3f9 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_service.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_service.cpp @@ -51,8 +51,8 @@ class TKqpQueryCache { AstIndex.emplace(GetQueryIdWithAst(*compileResult->Query, *compileResult->Ast), compileResult->Uid); } - bool Insert(const TKqpCompileResult::TConstPtr& compileResult, bool isEnableAstCache, bool isPerStatement) { - if (!isPerStatement) { + bool Insert(const TKqpCompileResult::TConstPtr& compileResult, bool isEnableAstCache, bool isPerStatementExecution) { + if (!isPerStatementExecution) { InsertQuery(compileResult); } if (isEnableAstCache && compileResult->Ast) { @@ -232,7 +232,7 @@ class TKqpQueryCache { }; struct TKqpCompileRequest { - TKqpCompileRequest(const TActorId& sender, const TString& uid, TKqpQueryId query, bool keepInCache, bool canDevideIntoStatements, + TKqpCompileRequest(const TActorId& sender, const TString& uid, TKqpQueryId query, bool keepInCache, bool canDivideIntoStatements, const TIntrusiveConstPtr& userToken, const TInstant& deadline, TKqpDbCountersPtr dbCounters, ui64 cookie, std::shared_ptr> intrestedInResult, const TIntrusivePtr& userRequestContext, @@ -244,7 +244,7 @@ struct TKqpCompileRequest { , Query(std::move(query)) , Uid(uid) , KeepInCache(keepInCache) - , CanDevideIntoStatements(canDevideIntoStatements) + , CanDivideIntoStatements(canDivideIntoStatements) , UserToken(userToken) , Deadline(deadline) , DbCounters(dbCounters) @@ -262,7 +262,7 @@ struct TKqpCompileRequest { TKqpQueryId Query; TString Uid; bool KeepInCache = false; - bool CanDevideIntoStatements = false; + bool CanDivideIntoStatements = false; TIntrusiveConstPtr UserToken; TInstant Deadline; TKqpDbCountersPtr DbCounters; @@ -638,7 +638,7 @@ class TKqpCompileService : public TActorBootstrapped { ev->Get()->Query ? ev->Get()->Query->UserSid : 0); TKqpCompileRequest compileRequest(ev->Sender, CreateGuidAsString(), std::move(*request.Query), - request.KeepInCache, request.CanDevideIntoStatements, request.UserToken, request.Deadline, dbCounters, + request.KeepInCache, request.CanDivideIntoStatements, request.UserToken, request.Deadline, dbCounters, ev->Cookie, std::move(ev->Get()->IntrestedInResult), ev->Get()->UserRequestContext, std::move(ev->Get()->Orbit), std::move(compileServiceSpan), std::move(ev->Get()->TempTablesState), TableServiceConfig.GetEnableAstCache() ? ECompileActorAction::PARSE : ECompileActorAction::COMPILE); @@ -756,14 +756,14 @@ class TKqpCompileService : public TActorBootstrapped { << ", compileActor: " << ev->Sender); bool keepInCache = compileRequest.KeepInCache && compileResult->AllowCache; - bool isPerStatement = TableServiceConfig.GetEnableAstCache() && compileRequest.QueryAst; + bool isPerStatementExecution = TableServiceConfig.GetEnableAstCache() && compileRequest.QueryAst; bool hasTempTablesNameClashes = HasTempTablesNameClashes(compileResult, compileRequest.TempTablesState, true); try { if (compileResult->Status == Ydb::StatusIds::SUCCESS) { if (!hasTempTablesNameClashes) { - UpdateQueryCache(compileResult, keepInCache, isPerStatement); + UpdateQueryCache(compileResult, keepInCache, isPerStatementExecution); } if (ev->Get()->ReplayMessage) { @@ -844,15 +844,15 @@ class TKqpCompileService : public TActorBootstrapped { return compileResult->PreparedQuery->HasTempTables(tempTablesState, withSessionId); } - void UpdateQueryCache(TKqpCompileResult::TConstPtr compileResult, bool keepInCache, bool isPerStatement) { + void UpdateQueryCache(TKqpCompileResult::TConstPtr compileResult, bool keepInCache, bool isPerStatementExecution) { if (QueryCache.FindByUid(compileResult->Uid, false)) { QueryCache.Replace(compileResult); } else if (keepInCache) { - if (QueryCache.Insert(compileResult, TableServiceConfig.GetEnableAstCache(), isPerStatement)) { + if (QueryCache.Insert(compileResult, TableServiceConfig.GetEnableAstCache(), isPerStatementExecution)) { Counters->CompileQueryCacheEvicted->Inc(); } if (compileResult->Query && compileResult->Query->Settings.IsPrepareQuery) { - if (InsertPreparingQuery(compileResult, true, isPerStatement)) { + if (InsertPreparingQuery(compileResult, true, isPerStatementExecution)) { Counters->CompileQueryCacheEvicted->Inc(); }; } @@ -925,7 +925,7 @@ class TKqpCompileService : public TActorBootstrapped { } private: - bool InsertPreparingQuery(const TKqpCompileResult::TConstPtr& compileResult, bool keepInCache, bool isPerStatement) { + bool InsertPreparingQuery(const TKqpCompileResult::TConstPtr& compileResult, bool keepInCache, bool isPerStatementExecution) { YQL_ENSURE(compileResult->Query); auto query = *compileResult->Query; @@ -950,7 +950,7 @@ class TKqpCompileService : public TActorBootstrapped { auto newCompileResult = TKqpCompileResult::Make(CreateGuidAsString(), compileResult->Status, compileResult->Issues, compileResult->MaxReadType, std::move(query), compileResult->Ast); newCompileResult->AllowCache = compileResult->AllowCache; newCompileResult->PreparedQuery = compileResult->PreparedQuery; - return QueryCache.Insert(newCompileResult, TableServiceConfig.GetEnableAstCache(), isPerStatement); + return QueryCache.Insert(newCompileResult, TableServiceConfig.GetEnableAstCache(), isPerStatementExecution); } void ProcessQueue(const TActorContext& ctx) { @@ -983,7 +983,7 @@ class TKqpCompileService : public TActorBootstrapped { void StartCompilation(TKqpCompileRequest&& request, const TActorContext& ctx) { auto compileActor = CreateKqpCompileActor(ctx.SelfID, KqpSettings, TableServiceConfig, QueryServiceConfig, MetadataProviderConfig, ModuleResolverState, Counters, request.Uid, request.Query, request.UserToken, FederatedQuerySetup, request.DbCounters, request.UserRequestContext, - request.CompileServiceSpan.GetTraceId(), request.TempTablesState, request.Action, std::move(request.QueryAst), CollectDiagnostics, request.CanDevideIntoStatements); + request.CompileServiceSpan.GetTraceId(), request.TempTablesState, request.Action, std::move(request.QueryAst), CollectDiagnostics, request.CanDivideIntoStatements); auto compileActorId = ctx.ExecutorThread.RegisterActor(compileActor, TMailboxType::HTSwap, AppData(ctx)->UserPoolId); diff --git a/ydb/core/kqp/compile_service/kqp_compile_service.h b/ydb/core/kqp/compile_service/kqp_compile_service.h index af37a4596e3d..c17d25c2bd5f 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_service.h +++ b/ydb/core/kqp/compile_service/kqp_compile_service.h @@ -38,7 +38,7 @@ IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstP ECompileActorAction compileAction = ECompileActorAction::COMPILE, TMaybe queryAst = {}, bool collectFullDiagnostics = false, - bool canDevideIntoStatements = false); + bool canDivideIntoStatements = false); IActor* CreateKqpCompileRequestActor(const TActorId& owner, const TIntrusiveConstPtr& userToken, const TMaybe& uid, TMaybe&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters, diff --git a/ydb/core/kqp/host/kqp_translate.cpp b/ydb/core/kqp/host/kqp_translate.cpp index 2804202040a2..a4fe1e15166b 100644 --- a/ydb/core/kqp/host/kqp_translate.cpp +++ b/ydb/core/kqp/host/kqp_translate.cpp @@ -56,7 +56,7 @@ NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type) { NSQLTranslation::TTranslationSettings GetTranslationSettings(NYql::EKikimrQueryType queryType, const TMaybe& usePgParser, bool sqlAutoCommit, const TString& queryText, std::shared_ptr> queryParameters, TMaybe& sqlVersion, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, - NYql::TExprContext& ctx, bool isEnablePgConstsToParams, bool perStatement) { + NYql::TExprContext& ctx, bool isEnablePgConstsToParams, bool perStatementExecution) { NSQLTranslation::TTranslationSettings settings{}; if (usePgParser) { @@ -145,7 +145,7 @@ NSQLTranslation::TTranslationSettings GetTranslationSettings(NYql::EKikimrQueryT } } - settings.PerStatement = perStatement; + settings.PerStatementExecution = perStatementExecution; return settings; } @@ -205,12 +205,12 @@ TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TM std::shared_ptr> queryParameters, bool sqlAutoCommit, TMaybe& sqlVersion, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, - NYql::TExprContext& ctx, bool isEnablePgConstsToParams, bool isSql, bool perStatement) { + NYql::TExprContext& ctx, bool isEnablePgConstsToParams, bool isSql, bool perStatementExecution) { TVector result; NYql::TAstParseResult astRes; if (isSql) { - auto settings = GetTranslationSettings(queryType, usePgParser, sqlAutoCommit, queryText, queryParameters, sqlVersion, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, ctx, isEnablePgConstsToParams, perStatement); + auto settings = GetTranslationSettings(queryType, usePgParser, sqlAutoCommit, queryText, queryParameters, sqlVersion, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, ctx, isEnablePgConstsToParams, perStatementExecution); ui16 actualSyntaxVersion = 0; auto astStatements = NSQLTranslation::SqlToAstStatements(queryText, settings, nullptr, &actualSyntaxVersion); sqlVersion = actualSyntaxVersion; @@ -226,7 +226,7 @@ TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TM TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TMaybe& syntax, const TString& queryText, std::shared_ptr> queryParameters, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, bool isEnablePgConstsToParams, bool isSql, - bool perStatement) { + bool perStatementExecution) { TMaybe sqlVersion; TMaybe usePgParser; if (syntax) @@ -248,7 +248,7 @@ TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TM } else { sqlAutoCommit = false; } - return SqlToAstStatements(queryType, usePgParser, queryText, queryParameters, sqlAutoCommit, sqlVersion, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, ctx, isEnablePgConstsToParams, isSql, perStatement); + return SqlToAstStatements(queryType, usePgParser, queryText, queryParameters, sqlAutoCommit, sqlVersion, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, ctx, isEnablePgConstsToParams, isSql, perStatementExecution); } } // namespace NKqp diff --git a/ydb/core/kqp/host/kqp_translate.h b/ydb/core/kqp/host/kqp_translate.h index 44c7158bce8f..0b18914b6845 100644 --- a/ydb/core/kqp/host/kqp_translate.h +++ b/ydb/core/kqp/host/kqp_translate.h @@ -13,7 +13,7 @@ NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type); NSQLTranslation::TTranslationSettings GetTranslationSettings(NYql::EKikimrQueryType queryType, const TMaybe& usePgParser, bool sqlAutoCommit, const TString& queryText, std::shared_ptr> queryParameters, TMaybe& sqlVersion, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, NYql::TExprContext& ctx, - bool isEnablePgConstsToParams, bool perStatement = false); + bool isEnablePgConstsToParams, bool perStatementExecution = false); NYql::TAstParseResult ParseQuery(NYql::EKikimrQueryType queryType, const TMaybe& usePgParser, const TString& queryText, std::shared_ptr> queryParameters, bool isSql, bool sqlAutoCommit, TMaybe& sqlVersion, @@ -25,7 +25,7 @@ TQueryAst ParseQuery(NYql::EKikimrQueryType queryType, const TMaybe SqlToAstStatements(NYql::EKikimrQueryType queryType, const TMaybe& syntax, const TString& queryText, std::shared_ptr> queryParameters, - TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, bool isEnablePgConstsToParams, bool isSql, bool perStatement); + TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, bool isEnablePgConstsToParams, bool isSql, bool perStatementExecution); } // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index d5cb17ade022..c7ae3179a204 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -162,7 +162,7 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi bool EnableAstCache = false; bool EnablePgConstsToParams = false; ui64 ExtractPredicateRangesLimit = 0; - bool EnableQueriesPerStatement = false; + bool EnablePerStatementQueryExecution = false; }; } diff --git a/ydb/core/kqp/session_actor/kqp_query_state.cpp b/ydb/core/kqp/session_actor/kqp_query_state.cpp index 0f7f6e3a3a41..5cf4dbb7f06f 100644 --- a/ydb/core/kqp/session_actor/kqp_query_state.cpp +++ b/ydb/core/kqp/session_actor/kqp_query_state.cpp @@ -140,7 +140,7 @@ std::unique_ptr TKqpQueryState::BuildCompileRequest(s settings.IsPrepareQuery = GetAction() == NKikimrKqp::QUERY_ACTION_PREPARE; bool keepInCache = false; - bool canDevideIntoStatements = !HasTxControl(); + bool canDivideIntoStatements = !HasTxControl(); switch (GetAction()) { case NKikimrKqp::QUERY_ACTION_EXECUTE: query = TKqpQueryId(Cluster, Database, GetQuery(), settings, GetQueryParameterTypes()); @@ -150,19 +150,19 @@ std::unique_ptr TKqpQueryState::BuildCompileRequest(s case NKikimrKqp::QUERY_ACTION_PREPARE: query = TKqpQueryId(Cluster, Database, GetQuery(), settings, GetQueryParameterTypes()); keepInCache = query->IsSql(); - canDevideIntoStatements = false; + canDivideIntoStatements = false; break; case NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED: uid = GetPreparedQuery(); keepInCache = GetQueryKeepInCache(); - canDevideIntoStatements = false; + canDivideIntoStatements = false; break; case NKikimrKqp::QUERY_ACTION_EXPLAIN: query = TKqpQueryId(Cluster, Database, GetQuery(), settings, GetQueryParameterTypes()); keepInCache = false; - canDevideIntoStatements = false; + canDivideIntoStatements = false; break; default: @@ -180,7 +180,7 @@ std::unique_ptr TKqpQueryState::BuildCompileRequest(s } return std::make_unique(UserToken, uid, std::move(query), keepInCache, - canDevideIntoStatements, compileDeadline, DbCounters, std::move(cookie), UserRequestContext, + canDivideIntoStatements, compileDeadline, DbCounters, std::move(cookie), UserRequestContext, std::move(Orbit), TempTablesState, GetCollectDiagnostics(), statementAst); } diff --git a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp index 4f3cc59613b5..629bceeaab9e 100644 --- a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp +++ b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp @@ -3680,7 +3680,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { } appConfig.MutableTableServiceConfig()->SetEnablePgConstsToParams(true); - appConfig.MutableTableServiceConfig()->SetEnableQueriesPerStatement(true); + appConfig.MutableTableServiceConfig()->SetEnablePerStatementQueryExecution(true); auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetAppConfig(appConfig) diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index 404423bfc040..0b82d7d7e0e0 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -1769,7 +1769,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { NKikimrConfig::TAppConfig appConfig; appConfig.MutableTableServiceConfig()->SetEnablePreparedDdl(true); appConfig.MutableTableServiceConfig()->SetEnableAstCache(true); - appConfig.MutableTableServiceConfig()->SetEnableQueriesPerStatement(true); + appConfig.MutableTableServiceConfig()->SetEnablePerStatementQueryExecution(true); auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetAppConfig(appConfig) @@ -1800,7 +1800,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 1); CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", FormatResultSetYson(result.GetResultSet(0))); - + result = db.ExecuteQuery(R"( UPSERT INTO TestDdl1 (Key, Value) VALUES (3, "Three"); )", TTxControl::NoTx()).ExtractValueSync(); @@ -1879,9 +1879,10 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { PRIMARY KEY (Key) ); UPSERT INTO TestDdl4 (Key, Val) VALUES (1, 1); + SELECT * FROM TestDdl4; )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - // NEED TO FIX IT: RESULT MUST BE DEVIDED INTO 2 SETS + // NEED TO FIX IT: RESULT MUST BE DIVIDED INTO 2 SETS UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 1); CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]];[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]];[[4u];["Four"]]])", FormatResultSetYson(result.GetResultSet(0))); //CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]]])", FormatResultSetYson(result.GetResultSet(0))); diff --git a/ydb/core/protos/table_service_config.proto b/ydb/core/protos/table_service_config.proto index 1a4cf4d0c1b5..05a913f54e59 100644 --- a/ydb/core/protos/table_service_config.proto +++ b/ydb/core/protos/table_service_config.proto @@ -275,5 +275,5 @@ message TTableServiceConfig { optional bool EnableOlapSink = 55 [default = false]; - optional bool EnableQueriesPerStatement = 56 [default = false]; + optional bool EnablePerStatementQueryExecution = 56 [default = false]; }; diff --git a/ydb/library/yql/sql/pg/pg_sql.cpp b/ydb/library/yql/sql/pg/pg_sql.cpp index 22a55bba11f6..3b268d799261 100644 --- a/ydb/library/yql/sql/pg/pg_sql.cpp +++ b/ydb/library/yql/sql/pg/pg_sql.cpp @@ -328,7 +328,7 @@ class TConverter : public IPGParseEvents { } void OnResult(const List* raw, int StatementId) { - if (!Settings.PerStatement) { + if (!Settings.PerStatementExecution) { return OnResult(raw); } @@ -4483,8 +4483,6 @@ NYql::TAstParseResult PGToYql(const TString& query, const NSQLTranslation::TTran TVector PGToYqlStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings) { TVector results; - size_t currentStatementId = 0; - bool finished; List* raw = NYql::PGGetStatements(query); if (!raw) { return {}; diff --git a/ydb/library/yql/sql/settings/translation_settings.h b/ydb/library/yql/sql/settings/translation_settings.h index 4667d70d6cd6..f4d0ea407314 100644 --- a/ydb/library/yql/sql/settings/translation_settings.h +++ b/ydb/library/yql/sql/settings/translation_settings.h @@ -107,7 +107,7 @@ namespace NSQLTranslation { bool AssumeYdbOnClusterWithSlash; TString DynamicClusterProvider; TString FileAliasPrefix; - bool PerStatement = false; + bool PerStatementExecution = false; TVector PgParameterTypeOids; bool AutoParametrizeEnabled = false; diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index 3cf44e61515b..9a088a4e83b2 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -144,7 +144,7 @@ TVector SqlToAstStatements(const TString& query, const NS if (astProto) { auto ast = static_cast(*astProto); const auto& query = ast.GetRule_sql_query(); - if (!settings.PerStatement) { + if (!settings.PerStatementExecution) { res.emplace_back(); SqlASTToYqlImpl(res.back(), *astProto, ctx); return res; diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index e98a1b74e2ef..d640235bf2ed 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -2721,17 +2721,6 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT_NO_DIFF(Err2Str(res), "
:1:13: Error: Unexpected character : syntax error...\n\n"); } - Y_UNIT_TEST(AAA) { - TTranslationSettings settings; - settings.PerStatement = true; - auto res = SqlToAstStatements("$a = (SELECT * FROM TestDdl1);\nSELECT * FROM $a;\nUPSERT INTO TestDdl1 (Key, Value) VALUES (3, \"Three\");", settings); - for (auto& i : res) { - if (!i.Root) { - Cerr << Err2Str(i); - } - } - } - Y_UNIT_TEST(InvalidDoubleAtStringWhichWasAcceptedEarlier) { NYql::TAstParseResult res = SqlToYql("SELECT @@foo@@ @ @@bar@@"); UNIT_ASSERT(!res.Root); From 361d86de48bbb55f2d6e71f045298b21917562fc Mon Sep 17 00:00:00 2001 From: Polina Volosnikova Date: Thu, 8 Feb 2024 08:05:41 +0000 Subject: [PATCH 3/5] fix result sets and issues --- ydb/core/kqp/common/compilation/events.h | 6 +-- .../kqp/compile_service/kqp_compile_actor.cpp | 23 ++++---- .../compile_service/kqp_compile_service.cpp | 10 ++-- .../kqp/compile_service/kqp_compile_service.h | 2 +- .../kqp/executer_actor/kqp_data_executer.cpp | 8 +-- ydb/core/kqp/executer_actor/kqp_executer.h | 2 +- .../kqp/executer_actor/kqp_executer_impl.cpp | 10 ++-- .../kqp/executer_actor/kqp_executer_impl.h | 11 ++-- .../kqp/executer_actor/kqp_result_channel.cpp | 12 +++-- .../kqp/executer_actor/kqp_result_channel.h | 2 +- .../kqp/executer_actor/kqp_scan_executer.cpp | 9 ++-- ydb/core/kqp/host/kqp_translate.cpp | 6 +-- ydb/core/kqp/host/kqp_translate.h | 2 +- .../kqp/session_actor/kqp_query_state.cpp | 10 ++-- ydb/core/kqp/session_actor/kqp_query_state.h | 6 ++- .../kqp/session_actor/kqp_session_actor.cpp | 13 ++++- ydb/core/kqp/ut/pg/kqp_pg_ut.cpp | 2 +- ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp | 54 ++++++++++++++----- ydb/core/protos/feature_flags.proto | 2 +- 19 files changed, 120 insertions(+), 70 deletions(-) diff --git a/ydb/core/kqp/common/compilation/events.h b/ydb/core/kqp/common/compilation/events.h index d5d632bfb8e0..ebb1311c6ade 100644 --- a/ydb/core/kqp/common/compilation/events.h +++ b/ydb/core/kqp/common/compilation/events.h @@ -14,7 +14,7 @@ namespace NKikimr::NKqp::NPrivateEvents { struct TEvCompileRequest: public TEventLocal { TEvCompileRequest(const TIntrusiveConstPtr& userToken, const TMaybe& uid, - TMaybe&& query, bool keepInCache, bool canDivideIntoStatements, TInstant deadline, + TMaybe&& query, bool keepInCache, bool perStatementResult, TInstant deadline, TKqpDbCountersPtr dbCounters, std::shared_ptr> intrestedInResult, const TIntrusivePtr& userRequestContext, NLWTrace::TOrbit orbit = {}, TKqpTempTablesState::TConstPtr tempTablesState = nullptr, bool collectDiagnostics = false, TMaybe queryAst = Nothing()) @@ -22,7 +22,7 @@ struct TEvCompileRequest: public TEventLocal Uid; TMaybe Query; bool KeepInCache = false; - bool CanDivideIntoStatements = false; + bool PerStatementResult = false; // it is allowed for local event to use absolute time (TInstant) instead of time interval (TDuration) TInstant Deadline; TKqpDbCountersPtr DbCounters; diff --git a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp index aea9765b12d4..a43ba0db2753 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp @@ -52,7 +52,7 @@ class TKqpCompileActor : public TActorBootstrapped { TKqpDbCountersPtr dbCounters, std::optional federatedQuerySetup, const TIntrusivePtr& userRequestContext, NWilson::TTraceId traceId, TKqpTempTablesState::TConstPtr tempTablesState, bool collectFullDiagnostics, - bool canDivideIntoStatements, ECompileActorAction compileAction, TMaybe queryAst) + bool perStatementResult, ECompileActorAction compileAction, TMaybe queryAst) : Owner(owner) , ModuleResolverState(moduleResolverState) , Counters(counters) @@ -70,7 +70,7 @@ class TKqpCompileActor : public TActorBootstrapped { , CompileActorSpan(TWilsonKqp::CompileActor, std::move(traceId), "CompileActor") , TempTablesState(std::move(tempTablesState)) , CollectFullDiagnostics(collectFullDiagnostics) - , CanDivideIntoStatements(canDivideIntoStatements) + , PerStatementResult(perStatementResult) , CompileAction(compileAction) , QueryAst(std::move(queryAst)) { @@ -136,9 +136,9 @@ class TKqpCompileActor : public TActorBootstrapped { NSQLTranslation::EBindingsMode bindingsMode = Config->BindingsMode; bool isEnableExternalDataSources = AppData(ctx)->FeatureFlags.GetEnableExternalDataSources(); bool isEnablePgConstsToParams = Config->EnablePgConstsToParams; - bool perStatementExecution = Config->EnablePerStatementQueryExecution && CanDivideIntoStatements; + bool perStatementExecution = Config->EnablePerStatementQueryExecution && PerStatementResult; - return SqlToAstStatements(ConvertType(QueryId.Settings.QueryType), QueryId.Settings.Syntax, QueryId.Text, QueryId.QueryParameterTypes, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, isEnablePgConstsToParams, QueryId.IsSql(), perStatementExecution); + return ParseStatements(ConvertType(QueryId.Settings.QueryType), QueryId.Settings.Syntax, QueryId.Text, QueryId.QueryParameterTypes, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, isEnablePgConstsToParams, QueryId.IsSql(), perStatementExecution); } void StartParsing(const TActorContext &ctx) { @@ -349,12 +349,13 @@ class TKqpCompileActor : public TActorBootstrapped { << ", at state:" << state); } - void ReplyParseResult(const TActorContext &ctx, TVector astStatements) { + void ReplyParseResult(const TActorContext &ctx, TVector&& astStatements) { Y_UNUSED(ctx); if (astStatements.empty()) { NYql::TIssue issue(NYql::TPosition(), "Parsing result of query is empty"); - ReplyError(Ydb::StatusIds::GENERIC_ERROR, {issue}); + ReplyError(Ydb::StatusIds::INTERNAL_ERROR, {issue}); + return; } for (size_t statementId = 0; statementId < astStatements.size(); ++statementId) { @@ -369,7 +370,7 @@ class TKqpCompileActor : public TActorBootstrapped { issue.AddSubIssue(MakeIntrusive(i)); } - ReplyError(Ydb::StatusIds::GENERIC_ERROR, {issue}); + ReplyError(Ydb::StatusIds::INTERNAL_ERROR, {issue}); return; } } @@ -379,7 +380,7 @@ class TKqpCompileActor : public TActorBootstrapped { << ", owner: " << Owner << ", statements size: " << astStatements.size()); - auto responseEv = MakeHolder(QueryId, astStatements); + auto responseEv = MakeHolder(QueryId, std::move(astStatements)); Send(Owner, responseEv.Release()); Counters->ReportCompileFinish(DbCounters); @@ -501,7 +502,7 @@ class TKqpCompileActor : public TActorBootstrapped { TKqpTempTablesState::TConstPtr TempTablesState; bool CollectFullDiagnostics; - const bool CanDivideIntoStatements; + const bool PerStatementResult; ECompileActorAction CompileAction; TMaybe QueryAst; }; @@ -549,14 +550,14 @@ IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstP TKqpDbCountersPtr dbCounters, const TIntrusivePtr& userRequestContext, NWilson::TTraceId traceId, TKqpTempTablesState::TConstPtr tempTablesState, ECompileActorAction compileAction, TMaybe queryAst, bool collectFullDiagnostics, - bool canDivideIntoStatements) + bool perStatementResult) { return new TKqpCompileActor(owner, kqpSettings, tableServiceConfig, queryServiceConfig, metadataProviderConfig, moduleResolverState, counters, uid, query, userToken, dbCounters, federatedQuerySetup, userRequestContext, std::move(traceId), std::move(tempTablesState), collectFullDiagnostics, - canDivideIntoStatements, compileAction, std::move(queryAst)); + perStatementResult, compileAction, std::move(queryAst)); } } // namespace NKqp diff --git a/ydb/core/kqp/compile_service/kqp_compile_service.cpp b/ydb/core/kqp/compile_service/kqp_compile_service.cpp index 45ed1c7bd3f9..97fc5b700935 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_service.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_service.cpp @@ -232,7 +232,7 @@ class TKqpQueryCache { }; struct TKqpCompileRequest { - TKqpCompileRequest(const TActorId& sender, const TString& uid, TKqpQueryId query, bool keepInCache, bool canDivideIntoStatements, + TKqpCompileRequest(const TActorId& sender, const TString& uid, TKqpQueryId query, bool keepInCache, bool perStatementResult, const TIntrusiveConstPtr& userToken, const TInstant& deadline, TKqpDbCountersPtr dbCounters, ui64 cookie, std::shared_ptr> intrestedInResult, const TIntrusivePtr& userRequestContext, @@ -244,7 +244,7 @@ struct TKqpCompileRequest { , Query(std::move(query)) , Uid(uid) , KeepInCache(keepInCache) - , CanDivideIntoStatements(canDivideIntoStatements) + , PerStatementResult(perStatementResult) , UserToken(userToken) , Deadline(deadline) , DbCounters(dbCounters) @@ -262,7 +262,7 @@ struct TKqpCompileRequest { TKqpQueryId Query; TString Uid; bool KeepInCache = false; - bool CanDivideIntoStatements = false; + bool PerStatementResult = false; TIntrusiveConstPtr UserToken; TInstant Deadline; TKqpDbCountersPtr DbCounters; @@ -638,7 +638,7 @@ class TKqpCompileService : public TActorBootstrapped { ev->Get()->Query ? ev->Get()->Query->UserSid : 0); TKqpCompileRequest compileRequest(ev->Sender, CreateGuidAsString(), std::move(*request.Query), - request.KeepInCache, request.CanDivideIntoStatements, request.UserToken, request.Deadline, dbCounters, + request.KeepInCache, request.PerStatementResult, request.UserToken, request.Deadline, dbCounters, ev->Cookie, std::move(ev->Get()->IntrestedInResult), ev->Get()->UserRequestContext, std::move(ev->Get()->Orbit), std::move(compileServiceSpan), std::move(ev->Get()->TempTablesState), TableServiceConfig.GetEnableAstCache() ? ECompileActorAction::PARSE : ECompileActorAction::COMPILE); @@ -983,7 +983,7 @@ class TKqpCompileService : public TActorBootstrapped { void StartCompilation(TKqpCompileRequest&& request, const TActorContext& ctx) { auto compileActor = CreateKqpCompileActor(ctx.SelfID, KqpSettings, TableServiceConfig, QueryServiceConfig, MetadataProviderConfig, ModuleResolverState, Counters, request.Uid, request.Query, request.UserToken, FederatedQuerySetup, request.DbCounters, request.UserRequestContext, - request.CompileServiceSpan.GetTraceId(), request.TempTablesState, request.Action, std::move(request.QueryAst), CollectDiagnostics, request.CanDivideIntoStatements); + request.CompileServiceSpan.GetTraceId(), request.TempTablesState, request.Action, std::move(request.QueryAst), CollectDiagnostics, request.PerStatementResult); auto compileActorId = ctx.ExecutorThread.RegisterActor(compileActor, TMailboxType::HTSwap, AppData(ctx)->UserPoolId); diff --git a/ydb/core/kqp/compile_service/kqp_compile_service.h b/ydb/core/kqp/compile_service/kqp_compile_service.h index c17d25c2bd5f..b2d4c43c7a71 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_service.h +++ b/ydb/core/kqp/compile_service/kqp_compile_service.h @@ -38,7 +38,7 @@ IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstP ECompileActorAction compileAction = ECompileActorAction::COMPILE, TMaybe queryAst = {}, bool collectFullDiagnostics = false, - bool canDivideIntoStatements = false); + bool PerStatementResult = false); IActor* CreateKqpCompileRequestActor(const TActorId& owner, const TIntrusiveConstPtr& userToken, const TMaybe& uid, TMaybe&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters, diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp index 0c7cdb69961c..543154266ae0 100644 --- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp @@ -128,9 +128,9 @@ class TKqpDataExecuter : public TKqpExecuterBase& userRequestContext, - const bool enableOlapSink) + const bool enableOlapSink, ui32 statementResultIndex) : TBase(std::move(request), database, userToken, counters, executerRetriesConfig, chanTransportVersion, aggregation, - maximalSecretsSnapshotWaitTime, userRequestContext, TWilsonKqp::DataExecuter, "DataExecuter", streamResult + maximalSecretsSnapshotWaitTime, userRequestContext, statementResultIndex, TWilsonKqp::DataExecuter, "DataExecuter", streamResult ) , AsyncIoFactory(std::move(asyncIoFactory)) , EnableOlapSink(enableOlapSink) @@ -2425,10 +2425,10 @@ IActor* CreateKqpDataExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TKqpRequestCounters::TPtr counters, bool streamResult, const NKikimrConfig::TTableServiceConfig::TAggregationConfig& aggregation, const NKikimrConfig::TTableServiceConfig::TExecuterRetriesConfig& executerRetriesConfig, NYql::NDq::IDqAsyncIoFactory::TPtr asyncIoFactory, const NKikimrConfig::TTableServiceConfig::EChannelTransportVersion chanTransportVersion, const TActorId& creator, - TDuration maximalSecretsSnapshotWaitTime, const TIntrusivePtr& userRequestContext, const bool enableOlapSink) + TDuration maximalSecretsSnapshotWaitTime, const TIntrusivePtr& userRequestContext, const bool enableOlapSink, ui32 statementResultIndex) { return new TKqpDataExecuter(std::move(request), database, userToken, counters, streamResult, executerRetriesConfig, - std::move(asyncIoFactory), chanTransportVersion, aggregation, creator, maximalSecretsSnapshotWaitTime, userRequestContext, enableOlapSink); + std::move(asyncIoFactory), chanTransportVersion, aggregation, creator, maximalSecretsSnapshotWaitTime, userRequestContext, enableOlapSink, statementResultIndex); } } // namespace NKqp diff --git a/ydb/core/kqp/executer_actor/kqp_executer.h b/ydb/core/kqp/executer_actor/kqp_executer.h index 242c64027224..bab811687132 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer.h +++ b/ydb/core/kqp/executer_actor/kqp_executer.h @@ -92,7 +92,7 @@ IActor* CreateKqpExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TSt NYql::NDq::IDqAsyncIoFactory::TPtr asyncIoFactory, TPreparedQueryHolder::TConstPtr preparedQuery, const NKikimrConfig::TTableServiceConfig::EChannelTransportVersion chanTransportVersion, const TActorId& creator, TDuration maximalSecretsSnapshotWaitTime, const TIntrusivePtr& userRequestContext, - const bool enableOlapSink); + const bool enableOlapSink, ui32 statementResultIndex); IActor* CreateKqpSchemeExecuter( TKqpPhyTxHolder::TConstPtr phyTx, NKikimrKqp::EQueryType queryType, const TActorId& target, diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.cpp b/ydb/core/kqp/executer_actor/kqp_executer_impl.cpp index b3aa7fd98b6d..2262fe3d6295 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_impl.cpp +++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.cpp @@ -83,12 +83,12 @@ IActor* CreateKqpExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TSt NYql::NDq::IDqAsyncIoFactory::TPtr asyncIoFactory, TPreparedQueryHolder::TConstPtr preparedQuery, const NKikimrConfig::TTableServiceConfig::EChannelTransportVersion chanTransportVersion, const TActorId& creator, TDuration maximalSecretsSnapshotWaitTime, const TIntrusivePtr& userRequestContext, - const bool enableOlapSink) + const bool enableOlapSink, ui32 statementResultIndex) { if (request.Transactions.empty()) { // commit-only or rollback-only data transaction YQL_ENSURE(request.LocksOp == ELocksOp::Commit || request.LocksOp == ELocksOp::Rollback); - return CreateKqpDataExecuter(std::move(request), database, userToken, counters, false, aggregation, executerRetriesConfig, std::move(asyncIoFactory), chanTransportVersion, creator, maximalSecretsSnapshotWaitTime, userRequestContext, enableOlapSink); + return CreateKqpDataExecuter(std::move(request), database, userToken, counters, false, aggregation, executerRetriesConfig, std::move(asyncIoFactory), chanTransportVersion, creator, maximalSecretsSnapshotWaitTime, userRequestContext, enableOlapSink, statementResultIndex); } TMaybe txsType; @@ -104,13 +104,13 @@ IActor* CreateKqpExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TSt switch (*txsType) { case NKqpProto::TKqpPhyTx::TYPE_COMPUTE: case NKqpProto::TKqpPhyTx::TYPE_DATA: - return CreateKqpDataExecuter(std::move(request), database, userToken, counters, false, aggregation, executerRetriesConfig, std::move(asyncIoFactory), chanTransportVersion, creator, maximalSecretsSnapshotWaitTime, userRequestContext, enableOlapSink); + return CreateKqpDataExecuter(std::move(request), database, userToken, counters, false, aggregation, executerRetriesConfig, std::move(asyncIoFactory), chanTransportVersion, creator, maximalSecretsSnapshotWaitTime, userRequestContext, enableOlapSink, statementResultIndex); case NKqpProto::TKqpPhyTx::TYPE_SCAN: - return CreateKqpScanExecuter(std::move(request), database, userToken, counters, aggregation, executerRetriesConfig, preparedQuery, chanTransportVersion, maximalSecretsSnapshotWaitTime, userRequestContext); + return CreateKqpScanExecuter(std::move(request), database, userToken, counters, aggregation, executerRetriesConfig, preparedQuery, chanTransportVersion, maximalSecretsSnapshotWaitTime, userRequestContext, statementResultIndex); case NKqpProto::TKqpPhyTx::TYPE_GENERIC: - return CreateKqpDataExecuter(std::move(request), database, userToken, counters, true, aggregation, executerRetriesConfig, std::move(asyncIoFactory), chanTransportVersion, creator, maximalSecretsSnapshotWaitTime, userRequestContext, enableOlapSink); + return CreateKqpDataExecuter(std::move(request), database, userToken, counters, true, aggregation, executerRetriesConfig, std::move(asyncIoFactory), chanTransportVersion, creator, maximalSecretsSnapshotWaitTime, userRequestContext, enableOlapSink, statementResultIndex); default: YQL_ENSURE(false, "Unsupported physical tx type: " << (ui32)*txsType); diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.h b/ydb/core/kqp/executer_actor/kqp_executer_impl.h index b0dda13154c9..c34f9474fe42 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_impl.h +++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.h @@ -122,7 +122,7 @@ class TKqpExecuterBase : public TActorBootstrapped { const NKikimrConfig::TTableServiceConfig::EChannelTransportVersion chanTransportVersion, const NKikimrConfig::TTableServiceConfig::TAggregationConfig& aggregation, TDuration maximalSecretsSnapshotWaitTime, const TIntrusivePtr& userRequestContext, - ui64 spanVerbosity = 0, TString spanName = "KqpExecuterBase", bool streamResult = false) + ui32 statementResultIndex, ui64 spanVerbosity = 0, TString spanName = "KqpExecuterBase", bool streamResult = false) : Request(std::move(request)) , Database(database) , UserToken(userToken) @@ -134,6 +134,7 @@ class TKqpExecuterBase : public TActorBootstrapped { , AggregationSettings(aggregation) , HasOlapTable(false) , StreamResult(streamResult) + , StatementResultIndex(statementResultIndex) { TasksGraph.GetMeta().Snapshot = IKqpGateway::TKqpSnapshot(Request.Snapshot.Step, Request.Snapshot.TxId); TasksGraph.GetMeta().Arena = MakeIntrusive(); @@ -1763,7 +1764,7 @@ class TKqpExecuterBase : public TActorBootstrapped { IActor* proxy; if (txResult.IsStream && txResult.QueryResultIndex.Defined()) { proxy = CreateResultStreamChannelProxy(TxId, channel.Id, txResult.MkqlItemType, - txResult.ColumnOrder, *txResult.QueryResultIndex, Target, this->SelfId()); + txResult.ColumnOrder, *txResult.QueryResultIndex, Target, this->SelfId(), StatementResultIndex); } else { proxy = CreateResultDataChannelProxy(TxId, channel.Id, this->SelfId(), channel.DstInputIndex, ResponseEv.get()); @@ -1899,6 +1900,8 @@ class TKqpExecuterBase : public TActorBootstrapped { THashMap ResultChannelToComputeActor; THashMap> SourceScanStageIdToParititions; + ui32 StatementResultIndex; + private: static constexpr TDuration ResourceUsageUpdateInterval = TDuration::MilliSeconds(100); }; @@ -1912,14 +1915,14 @@ IActor* CreateKqpDataExecuter(IKqpGateway::TExecPhysicalRequest&& request, const NYql::NDq::IDqAsyncIoFactory::TPtr asyncIoFactory, const NKikimrConfig::TTableServiceConfig::EChannelTransportVersion chanTransportVersion, const TActorId& creator, TDuration maximalSecretsSnapshotWaitTime, const TIntrusivePtr& userRequestContext, - const bool enableOlapSink); + const bool enableOlapSink, ui32 statementResultIndex); IActor* CreateKqpScanExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TIntrusiveConstPtr& userToken, TKqpRequestCounters::TPtr counters, const NKikimrConfig::TTableServiceConfig::TAggregationConfig& aggregation, const NKikimrConfig::TTableServiceConfig::TExecuterRetriesConfig& executerRetriesConfig, TPreparedQueryHolder::TConstPtr preparedQuery, const NKikimrConfig::TTableServiceConfig::EChannelTransportVersion chanTransportVersion, - TDuration maximalSecretsSnapshotWaitTime, const TIntrusivePtr& userRequestContext); + TDuration maximalSecretsSnapshotWaitTime, const TIntrusivePtr& userRequestContext, ui32 statementResultIndex); } // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/kqp/executer_actor/kqp_result_channel.cpp b/ydb/core/kqp/executer_actor/kqp_result_channel.cpp index 3d3d9d00d199..f2538e598a10 100644 --- a/ydb/core/kqp/executer_actor/kqp_result_channel.cpp +++ b/ydb/core/kqp/executer_actor/kqp_result_channel.cpp @@ -136,12 +136,13 @@ class TResultStreamChannelProxy : public TResultCommonChannelProxy { public: TResultStreamChannelProxy(ui64 txId, ui64 channelId, NKikimr::NMiniKQL::TType* itemType, const TVector* columnOrder, ui32 queryResultIndex, TActorId target, - TActorId executer) + TActorId executer, size_t statementResultIndex) : TResultCommonChannelProxy(txId, channelId, executer) , ColumnOrder(columnOrder) , ItemType(itemType) , QueryResultIndex(queryResultIndex) - , Target(target) {} + , Target(target) + , StatementResultIndex(statementResultIndex) {} private: void SendResults(TEvComputeChannelDataOOB& computeData, TActorId sender) override { @@ -158,7 +159,7 @@ class TResultStreamChannelProxy : public TResultCommonChannelProxy { auto streamEv = MakeHolder(); streamEv->Record.SetSeqNo(computeData.Proto.GetSeqNo()); - streamEv->Record.SetQueryResultIndex(QueryResultIndex); + streamEv->Record.SetQueryResultIndex(QueryResultIndex + StatementResultIndex); streamEv->Record.MutableResultSet()->Swap(&resultSet); LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, @@ -173,6 +174,7 @@ class TResultStreamChannelProxy : public TResultCommonChannelProxy { NKikimr::NMiniKQL::TType* ItemType; ui32 QueryResultIndex = 0; const NActors::TActorId Target; + size_t StatementResultIndex; }; class TResultDataChannelProxy : public TResultCommonChannelProxy { @@ -211,7 +213,7 @@ class TResultDataChannelProxy : public TResultCommonChannelProxy { NActors::IActor* CreateResultStreamChannelProxy(ui64 txId, ui64 channelId, NKikimr::NMiniKQL::TType* itemType, const TVector* columnOrder, ui32 queryResultIndex, TActorId target, - TActorId executer) + TActorId executer, ui32 statementResultIndex) { LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, "CreateResultStreamChannelProxy: TxId: " << txId << @@ -219,7 +221,7 @@ NActors::IActor* CreateResultStreamChannelProxy(ui64 txId, ui64 channelId, NKiki ); return new TResultStreamChannelProxy(txId, channelId, itemType, columnOrder, queryResultIndex, target, - executer); + executer, statementResultIndex); } NActors::IActor* CreateResultDataChannelProxy(ui64 txId, ui64 channelId, TActorId executer, diff --git a/ydb/core/kqp/executer_actor/kqp_result_channel.h b/ydb/core/kqp/executer_actor/kqp_result_channel.h index 1eddf0d033e1..e1820ea015bf 100644 --- a/ydb/core/kqp/executer_actor/kqp_result_channel.h +++ b/ydb/core/kqp/executer_actor/kqp_result_channel.h @@ -27,7 +27,7 @@ struct TKqpExecuterTxResult; NActors::IActor* CreateResultStreamChannelProxy(ui64 txId, ui64 channelId, NKikimr::NMiniKQL::TType* itemType, const TVector* columnOrder, ui32 queryResultIndex, NActors::TActorId target, - NActors::TActorId executer); + NActors::TActorId executer, ui32 statementResultIndex); NActors::IActor* CreateResultDataChannelProxy(ui64 txId, ui64 channelId, NActors::TActorId executer, ui32 inputIndex, TEvKqpExecuter::TEvTxResponse* receiver); diff --git a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp index d6f6f1e9813b..20196e26da08 100644 --- a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp @@ -49,9 +49,10 @@ class TKqpScanExecuter : public TKqpExecuterBase& userRequestContext) + TDuration maximalSecretsSnapshotWaitTime, const TIntrusivePtr& userRequestContext, + ui32 statementResultIndex) : TBase(std::move(request), database, userToken, counters, executerRetriesConfig, chanTransportVersion, aggregation, - maximalSecretsSnapshotWaitTime, userRequestContext, TWilsonKqp::ScanExecuter, "ScanExecuter", + maximalSecretsSnapshotWaitTime, userRequestContext, statementResultIndex, TWilsonKqp::ScanExecuter, "ScanExecuter", false ) , PreparedQuery(preparedQuery) @@ -363,10 +364,10 @@ IActor* CreateKqpScanExecuter(IKqpGateway::TExecPhysicalRequest&& request, const const NKikimrConfig::TTableServiceConfig::TAggregationConfig& aggregation, const NKikimrConfig::TTableServiceConfig::TExecuterRetriesConfig& executerRetriesConfig, TPreparedQueryHolder::TConstPtr preparedQuery, const NKikimrConfig::TTableServiceConfig::EChannelTransportVersion chanTransportVersion, - TDuration maximalSecretsSnapshotWaitTime, const TIntrusivePtr& userRequestContext) + TDuration maximalSecretsSnapshotWaitTime, const TIntrusivePtr& userRequestContext, ui32 statementResultIndex) { return new TKqpScanExecuter(std::move(request), database, userToken, counters, aggregation, executerRetriesConfig, - preparedQuery, chanTransportVersion, maximalSecretsSnapshotWaitTime, userRequestContext); + preparedQuery, chanTransportVersion, maximalSecretsSnapshotWaitTime, userRequestContext, statementResultIndex); } } // namespace NKqp diff --git a/ydb/core/kqp/host/kqp_translate.cpp b/ydb/core/kqp/host/kqp_translate.cpp index a4fe1e15166b..1d87b74abd85 100644 --- a/ydb/core/kqp/host/kqp_translate.cpp +++ b/ydb/core/kqp/host/kqp_translate.cpp @@ -201,7 +201,7 @@ TQueryAst ParseQuery(NYql::EKikimrQueryType queryType, const TMaybe(std::move(astRes)), sqlVersion, deprecatedSQL); } -TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TMaybe& usePgParser, const TString& queryText, +TVector ParseStatements(NYql::EKikimrQueryType queryType, const TMaybe& usePgParser, const TString& queryText, std::shared_ptr> queryParameters, bool sqlAutoCommit, TMaybe& sqlVersion, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, @@ -224,7 +224,7 @@ TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TM } } -TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TMaybe& syntax, const TString& queryText, std::shared_ptr> queryParameters, +TVector ParseStatements(NYql::EKikimrQueryType queryType, const TMaybe& syntax, const TString& queryText, std::shared_ptr> queryParameters, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, bool isEnablePgConstsToParams, bool isSql, bool perStatementExecution) { TMaybe sqlVersion; @@ -248,7 +248,7 @@ TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TM } else { sqlAutoCommit = false; } - return SqlToAstStatements(queryType, usePgParser, queryText, queryParameters, sqlAutoCommit, sqlVersion, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, ctx, isEnablePgConstsToParams, isSql, perStatementExecution); + return ParseStatements(queryType, usePgParser, queryText, queryParameters, sqlAutoCommit, sqlVersion, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, ctx, isEnablePgConstsToParams, isSql, perStatementExecution); } } // namespace NKqp diff --git a/ydb/core/kqp/host/kqp_translate.h b/ydb/core/kqp/host/kqp_translate.h index 0b18914b6845..1fbc54a77ca9 100644 --- a/ydb/core/kqp/host/kqp_translate.h +++ b/ydb/core/kqp/host/kqp_translate.h @@ -24,7 +24,7 @@ TQueryAst ParseQuery(NYql::EKikimrQueryType queryType, const TMaybe> queryParameters, bool isSql, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, bool isEnablePgConstsToParams); -TVector SqlToAstStatements(NYql::EKikimrQueryType queryType, const TMaybe& syntax, const TString& queryText, std::shared_ptr> queryParameters, +TVector ParseStatements(NYql::EKikimrQueryType queryType, const TMaybe& syntax, const TString& queryText, std::shared_ptr> queryParameters, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, bool isEnablePgConstsToParams, bool isSql, bool perStatementExecution); } // namespace NKqp diff --git a/ydb/core/kqp/session_actor/kqp_query_state.cpp b/ydb/core/kqp/session_actor/kqp_query_state.cpp index 5cf4dbb7f06f..707d852f64f2 100644 --- a/ydb/core/kqp/session_actor/kqp_query_state.cpp +++ b/ydb/core/kqp/session_actor/kqp_query_state.cpp @@ -140,7 +140,7 @@ std::unique_ptr TKqpQueryState::BuildCompileRequest(s settings.IsPrepareQuery = GetAction() == NKikimrKqp::QUERY_ACTION_PREPARE; bool keepInCache = false; - bool canDivideIntoStatements = !HasTxControl(); + bool perStatementResult = !HasTxControl(); switch (GetAction()) { case NKikimrKqp::QUERY_ACTION_EXECUTE: query = TKqpQueryId(Cluster, Database, GetQuery(), settings, GetQueryParameterTypes()); @@ -150,19 +150,19 @@ std::unique_ptr TKqpQueryState::BuildCompileRequest(s case NKikimrKqp::QUERY_ACTION_PREPARE: query = TKqpQueryId(Cluster, Database, GetQuery(), settings, GetQueryParameterTypes()); keepInCache = query->IsSql(); - canDivideIntoStatements = false; + perStatementResult = false; break; case NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED: uid = GetPreparedQuery(); keepInCache = GetQueryKeepInCache(); - canDivideIntoStatements = false; + perStatementResult = false; break; case NKikimrKqp::QUERY_ACTION_EXPLAIN: query = TKqpQueryId(Cluster, Database, GetQuery(), settings, GetQueryParameterTypes()); keepInCache = false; - canDivideIntoStatements = false; + perStatementResult = false; break; default: @@ -180,7 +180,7 @@ std::unique_ptr TKqpQueryState::BuildCompileRequest(s } return std::make_unique(UserToken, uid, std::move(query), keepInCache, - canDivideIntoStatements, compileDeadline, DbCounters, std::move(cookie), UserRequestContext, + perStatementResult, compileDeadline, DbCounters, std::move(cookie), UserRequestContext, std::move(Orbit), TempTablesState, GetCollectDiagnostics(), statementAst); } diff --git a/ydb/core/kqp/session_actor/kqp_query_state.h b/ydb/core/kqp/session_actor/kqp_query_state.h index b52117b9d969..140574192c06 100644 --- a/ydb/core/kqp/session_actor/kqp_query_state.h +++ b/ydb/core/kqp/session_actor/kqp_query_state.h @@ -126,7 +126,9 @@ class TKqpQueryState : public TNonCopyable { NYql::TIssues Issues; TVector Statements; - size_t CurrentStatementId = 0; + ui32 CurrentStatementId = 0; + ui32 StatementResultIndex = 0; + ui32 StatementResultSize = 0; NKikimrKqp::EQueryAction GetAction() const { return RequestEv->GetAction(); @@ -391,6 +393,8 @@ class TKqpQueryState : public TNonCopyable { void PrepareNextStatement() { CurrentStatementId++; + StatementResultIndex += StatementResultSize; + StatementResultSize = 0; PrepareCurrentStatement(); } diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index a69555932e86..69c59a4c6fea 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -79,7 +79,7 @@ std::optional TryDecodeYdbSessionId(const TString& sessionId) { #define LOG_W(msg) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) #define LOG_N(msg) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) #define LOG_I(msg) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) -#define LOG_D(msg) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) +#define LOG_D(msg) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) #define LOG_T(msg) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) void FillColumnsMeta(const NKqpProto::TKqpPhyQuery& phyQuery, NKikimrKqp::TQueryResponse& resp) { @@ -1116,9 +1116,18 @@ class TKqpSessionActor : public TActorBootstrapped { ExecuterId = RegisterWithSameMailbox(executerActor); } + static ui32 GetResultsCount(const IKqpGateway::TExecPhysicalRequest& req) { + ui32 results = 0; + for (const auto& transaction : req.Transactions) { + results += transaction.Body->ResultsSize(); + } + return results; + } + void SendToExecuter(IKqpGateway::TExecPhysicalRequest&& request, bool isRollback = false) { if (QueryState) { request.Orbit = std::move(QueryState->Orbit); + QueryState->StatementResultSize = GetResultsCount(request); } request.PerRequestDataSizeLimit = RequestControls.PerRequestDataSizeLimit; request.MaxShardCount = RequestControls.MaxShardCount; @@ -1130,7 +1139,7 @@ class TKqpSessionActor : public TActorBootstrapped { RequestCounters, Settings.TableService.GetAggregationConfig(), Settings.TableService.GetExecuterRetriesConfig(), AsyncIoFactory, QueryState ? QueryState->PreparedQuery : nullptr, Settings.TableService.GetChannelTransportVersion(), SelfId(), 2 * TDuration::Seconds(MetadataProviderConfig.GetRefreshPeriodSeconds()), QueryState ? QueryState->UserRequestContext : MakeIntrusive("", Settings.Database, SessionId), - Settings.TableService.GetEnableOlapSink()); + Settings.TableService.GetEnableOlapSink(), QueryState ? QueryState->StatementResultIndex : 0); auto exId = RegisterWithSameMailbox(executerActor); LOG_D("Created new KQP executer: " << exId << " isRollback: " << isRollback); diff --git a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp index 629bceeaab9e..89bb1bc299fe 100644 --- a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp +++ b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp @@ -3790,7 +3790,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { SELECT (1, 2); )"); auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::INTERNAL_ERROR, result.GetIssues().ToString()); UNIT_ASSERT(result.GetIssues().ToString().Contains("alternative is not implemented yet : 138")); } } diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index 0b82d7d7e0e0..ff65bee68ada 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -1779,7 +1779,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto db = kikimr.GetQueryClient(); { - // Base test with ddl and adm statements + // Base test with ddl and dml statements auto result = db.ExecuteQuery(R"( DECLARE $name AS Text; $a = (SELECT * FROM TestDdl1); @@ -1800,11 +1800,13 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 1); CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", FormatResultSetYson(result.GetResultSet(0))); + UNIT_ASSERT_EQUAL_C(result.GetIssues().Size(), 0, result.GetIssues().ToString()); result = db.ExecuteQuery(R"( UPSERT INTO TestDdl1 (Key, Value) VALUES (3, "Three"); )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_EQUAL_C(result.GetIssues().Size(), 0, result.GetIssues().ToString()); result = db.ExecuteQuery(R"( SELECT * FROM TestDdl1; @@ -1812,6 +1814,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 1); CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]]])", FormatResultSetYson(result.GetResultSet(0))); + UNIT_ASSERT_EQUAL_C(result.GetIssues().Size(), 0, result.GetIssues().ToString()); result = db.ExecuteQuery(R"( CREATE TABLE TestDdl1 ( @@ -1832,6 +1835,13 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); UNIT_ASSERT(result.GetIssues().ToOneLineString().Contains("Check failed: path: '/Root/TestDdl2', error: path exist")); + + result = db.ExecuteQuery(R"( + UPSERT INTO TestDdl2 SELECT * FROM TestDdl1; + )", TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 0); + UNIT_ASSERT_EQUAL_C(result.GetIssues().Size(), 0, result.GetIssues().ToString()); } { @@ -1843,7 +1853,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { Value String, PRIMARY KEY (Key) ); - UPSERT INTO TestDdl2 (Key, Value) VALUES (2, "Two"); + UPSERT INTO TestDdl2 (Key, Value) VALUES (4, "Four"); CREATE TABLE TestDdl2 ( Key Uint64, Value String, @@ -1854,20 +1864,22 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { Value String, PRIMARY KEY (Key) ); - UPSERT INTO TestDdl1 (Key, Value) VALUES (3, "Thee"); + UPSERT INTO TestDdl1 (Key, Value) VALUES (3, "Three"); )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 0); result = db.ExecuteQuery(R"( SELECT * FROM TestDdl2; )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 1); - CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", FormatResultSetYson(result.GetResultSet(0))); + CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]];[[4u];["Four"]]])", FormatResultSetYson(result.GetResultSet(0))); + UNIT_ASSERT_EQUAL_C(result.GetIssues().Size(), 0, result.GetIssues().ToString()); } { - // Test with several select + // Check result sets auto result = db.ExecuteQuery(R"( $a = (SELECT * FROM TestDdl1); SELECT * FROM $a; @@ -1875,18 +1887,36 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { SELECT * FROM $a; CREATE TABLE TestDdl4 ( Key Uint64, - Val Uint64, + Value Uint64, PRIMARY KEY (Key) ); - UPSERT INTO TestDdl4 (Key, Val) VALUES (1, 1); + UPSERT INTO TestDdl4 (Key, Value) VALUES (1, 1); SELECT * FROM TestDdl4; )", TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - // NEED TO FIX IT: RESULT MUST BE DIVIDED INTO 2 SETS - UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 1); - CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]];[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]];[[4u];["Four"]]])", FormatResultSetYson(result.GetResultSet(0))); - //CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]]])", FormatResultSetYson(result.GetResultSet(0))); - //CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]][[4u];["Four"]]])", FormatResultSetYson(result.GetResultSet(1))); + UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 3); + CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]]])", FormatResultSetYson(result.GetResultSet(0))); + CompareYson(R"([[[1u];["One"]];[[2u];["Two"]];[[3u];["Three"]];[[4u];["Four"]]])", FormatResultSetYson(result.GetResultSet(1))); + CompareYson(R"([[[1u];[1u]]])", FormatResultSetYson(result.GetResultSet(2))); + UNIT_ASSERT_EQUAL_C(result.GetIssues().Size(), 0, result.GetIssues().ToString()); + + result = db.ExecuteQuery(R"( + UPSERT INTO TestDdl2 SELECT * FROM TestDdl1; + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 0); + UNIT_ASSERT_EQUAL_C(result.GetIssues().Size(), 0, result.GetIssues().ToString()); + } + + { + // Check EVALUATE FOR + auto result = db.ExecuteQuery(R"( + EVALUATE FOR $i IN AsList(1, 2, 3) DO BEGIN + SELECT $i; + SELECT $i; + END DO; + )", TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::UNSUPPORTED, result.GetIssues().ToString()); } } } diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index ae6e590c731c..4e405b9f3c93 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -105,7 +105,7 @@ message TFeatureFlags { optional bool EnableSeparationComputeActorsFromRead = 90 [default = false]; optional bool EnablePQConfigTransactionsAtSchemeShard = 91 [default = false]; optional bool EnableScriptExecutionOperations = 92 [default = false]; - optional bool EnableImplicitQueryParameterTypes = 93 [default = false]; + optional bool EnableImplicitQueryParameterTypes = 93 [default = true]; optional bool EnableForceImmediateEffectsExecution = 94 [default = false]; optional bool EnableTopicSplitMerge = 95 [default = false]; optional bool EnableChangefeedDynamoDBStreamsFormat = 96 [default = true]; From e31bd61c522810646cd64e187452f52d1dc7537b Mon Sep 17 00:00:00 2001 From: Polina Volosnikova Date: Wed, 14 Feb 2024 23:43:00 +0000 Subject: [PATCH 4/5] fix result sets and fix using pg parser --- .../kqp/executer_actor/kqp_executer_impl.h | 2 +- ydb/core/kqp/host/kqp_translate.cpp | 3 + .../kqp/session_actor/kqp_session_actor.cpp | 4 +- ydb/library/yql/parser/pg_wrapper/parser.cpp | 29 -- ydb/library/yql/sql/pg/pg_sql.cpp | 325 ++++++++++-------- 5 files changed, 197 insertions(+), 166 deletions(-) diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.h b/ydb/core/kqp/executer_actor/kqp_executer_impl.h index c34f9474fe42..b030e5285e68 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_impl.h +++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.h @@ -287,7 +287,7 @@ class TKqpExecuterBase : public TActorBootstrapped { if (!trailingResults) { auto streamEv = MakeHolder(); streamEv->Record.SetSeqNo(computeData.Proto.GetSeqNo()); - streamEv->Record.SetQueryResultIndex(*txResult.QueryResultIndex); + streamEv->Record.SetQueryResultIndex(*txResult.QueryResultIndex + StatementResultIndex); streamEv->Record.SetChannelId(channel.Id); streamEv->Record.MutableResultSet()->Swap(&resultSet); diff --git a/ydb/core/kqp/host/kqp_translate.cpp b/ydb/core/kqp/host/kqp_translate.cpp index 1d87b74abd85..269419ed4133 100644 --- a/ydb/core/kqp/host/kqp_translate.cpp +++ b/ydb/core/kqp/host/kqp_translate.cpp @@ -227,6 +227,9 @@ TVector ParseStatements(NYql::EKikimrQueryType queryType, const TMayb TVector ParseStatements(NYql::EKikimrQueryType queryType, const TMaybe& syntax, const TString& queryText, std::shared_ptr> queryParameters, TString cluster, TString kqpTablePathPrefix, ui16 kqpYqlSyntaxVersion, NSQLTranslation::EBindingsMode bindingsMode, bool isEnableExternalDataSources, bool isEnablePgConstsToParams, bool isSql, bool perStatementExecution) { + if (!perStatementExecution) { + return {ParseQuery(queryType, syntax, queryText, queryParameters, isSql, cluster, kqpTablePathPrefix, kqpYqlSyntaxVersion, bindingsMode, isEnableExternalDataSources, isEnablePgConstsToParams)}; + } TMaybe sqlVersion; TMaybe usePgParser; if (syntax) diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index 69c59a4c6fea..197c13375603 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -79,7 +79,7 @@ std::optional TryDecodeYdbSessionId(const TString& sessionId) { #define LOG_W(msg) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) #define LOG_N(msg) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) #define LOG_I(msg) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) -#define LOG_D(msg) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) +#define LOG_D(msg) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) #define LOG_T(msg) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, LogPrefix() << msg) void FillColumnsMeta(const NKqpProto::TKqpPhyQuery& phyQuery, NKikimrKqp::TQueryResponse& resp) { @@ -796,7 +796,7 @@ class TKqpSessionActor : public TActorBootstrapped { request.QueryType = queryState->GetType(); if (Y_LIKELY(queryState->PreparedQuery)) { ui64 resultSetsCount = queryState->PreparedQuery->GetPhysicalQuery().ResultBindingsSize(); - request.AllowTrailingResults = (resultSetsCount == 1); + request.AllowTrailingResults = (resultSetsCount == 1 && queryState->Statements.size() <= 1); request.AllowTrailingResults &= (QueryState->RequestEv->GetSupportsStreamTrailingResult()); } } diff --git a/ydb/library/yql/parser/pg_wrapper/parser.cpp b/ydb/library/yql/parser/pg_wrapper/parser.cpp index 17c7990eefa4..73c82fe2f9e4 100644 --- a/ydb/library/yql/parser/pg_wrapper/parser.cpp +++ b/ydb/library/yql/parser/pg_wrapper/parser.cpp @@ -216,35 +216,6 @@ void PGParse(const TString& input, IPGParseEvents& events) { } } -List* PGGetStatements(const TString& input) { - pg_thread_init(); - - PgQueryInternalParsetreeAndError parsetree_and_error; - - TArenaMemoryContext arena; - auto prevErrorContext = ErrorContext; - ErrorContext = CurrentMemoryContext; - - Y_DEFER { - ErrorContext = prevErrorContext; - }; - - parsetree_and_error = pg_query_raw_parse(input.c_str()); - Y_DEFER { - if (parsetree_and_error.error) { - pg_query_free_error(parsetree_and_error.error); - } - - free(parsetree_and_error.stderr_buffer); - }; - - if (parsetree_and_error.error) { - return nullptr; - } else { - return parsetree_and_error.tree; - } -} - TString PrintPGTree(const List* raw) { auto str = nodeToString(raw); Y_DEFER { diff --git a/ydb/library/yql/sql/pg/pg_sql.cpp b/ydb/library/yql/sql/pg/pg_sql.cpp index 3b268d799261..587d1936975b 100644 --- a/ydb/library/yql/sql/pg/pg_sql.cpp +++ b/ydb/library/yql/sql/pg/pg_sql.cpp @@ -273,13 +273,24 @@ class TConverter : public IPGParseEvents { using TViews = THashMap; - TConverter(TAstParseResult& astParseResult, const NSQLTranslation::TTranslationSettings& settings, const TString& query) - : AstParseResult(astParseResult) + struct TState { + TString CostBasedOptimizer; + TVector Statements; + ui32 ReadIndex = 0; + TViews Views; + TVector CTE; + TVector Positions = {NYql::TPosition()}; + THashMap ParamNameToPgTypeName; + THashMap AutoParamValues; + }; + + TConverter(TVector& astParseResults, const NSQLTranslation::TTranslationSettings& settings, const TString& query) + : AstParseResults(astParseResults) , Settings(settings) , DqEngineEnabled(Settings.DqDefaultAuto->Allow()) , BlockEngineEnabled(Settings.BlockDefaultAuto->Allow()) { - Positions.push_back({}); + AstParseResults.push_back({}); ScanRows(query); for (auto& flag : Settings.Flags) { @@ -314,16 +325,28 @@ class TConverter : public IPGParseEvents { const auto typeOid = Settings.PgParameterTypeOids[i]; const auto& typeName = typeOid != UNKNOWNOID ? NPg::LookupType(typeOid).Name : DEFAULT_PARAM_TYPE; - ParamNameToPgTypeName[paramName] = typeName; + State.ParamNameToPgTypeName[paramName] = typeName; } } void OnResult(const List* raw) { - AstParseResult.Pool = std::make_unique(4096); - AstParseResult.Root = ParseResult(raw); - if (!AutoParamValues.empty()) { - AstParseResult.PgAutoParamValues = std::move(AutoParamValues); + if (!Settings.PerStatementExecution) { + AstParseResults[StatementId].Pool = std::make_unique(4096); + AstParseResults[StatementId].Root = ParseResult(raw); + if (!State.AutoParamValues.empty()) { + AstParseResults[StatementId].PgAutoParamValues = std::move(State.AutoParamValues); + } + return; + } + AstParseResults.resize(ListLength(raw)); + for (; StatementId < AstParseResults.size(); ++StatementId) { + AstParseResults[StatementId].Pool = std::make_unique(4096); + AstParseResults[StatementId].Root = ParseResult(raw, StatementId); + if (!State.AutoParamValues.empty()) { + AstParseResults[StatementId].PgAutoParamValues = std::move(State.AutoParamValues); + } + State = {}; } } @@ -332,77 +355,125 @@ class TConverter : public IPGParseEvents { return OnResult(raw); } - AstParseResult.Pool = std::make_unique(4096); - AstParseResult.Root = ParseResult(raw, StatementId); - if (!AutoParamValues.empty()) { - AstParseResult.PgAutoParamValues = std::move(AutoParamValues); + AstParseResults[StatementId].Pool = std::make_unique(4096); + AstParseResults[StatementId].Root = ParseResult(raw, StatementId); + if (!State.AutoParamValues.empty()) { + AstParseResults[StatementId].PgAutoParamValues = std::move(State.AutoParamValues); } } void OnError(const TIssue& issue) { - AstParseResult.Issues.AddIssue(issue); + AstParseResults[StatementId].Issues.AddIssue(issue); } - TAstNode* ParseResult(const List* raw, const TMaybe& statementId = Nothing()) { + TAstNode* ParseResult(const List* raw, int statementId) { auto configSource = L(A("DataSource"), QA(TString(NYql::ConfigProviderName))); - Statements.push_back(L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, + State.Statements.push_back(L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, QA("OrderedColumns")))); - ui32 blockEnginePgmPos = Statements.size(); - Statements.push_back(configSource); - ui32 costBasedOptimizerPos = Statements.size(); - Statements.push_back(configSource); - ui32 dqEnginePgmPos = Statements.size(); - Statements.push_back(configSource); + ui32 blockEnginePgmPos = State.Statements.size(); + State.Statements.push_back(configSource); + ui32 costBasedOptimizerPos = State.Statements.size(); + State.Statements.push_back(configSource); + ui32 dqEnginePgmPos = State.Statements.size(); + State.Statements.push_back(configSource); - if (statementId) { - if (!ParseRawStmt(LIST_CAST_NTH(RawStmt, raw, *statementId))) { - return nullptr; - } + if (!ParseRawStmt(LIST_CAST_NTH(RawStmt, raw, statementId))) { + return nullptr; + } + + if (!State.Views.empty()) { + AddError("Not all views have been dropped"); + return nullptr; + } + + if (Settings.EndOfQueryCommit) { + State.Statements.push_back(L(A("let"), A("world"), L(A("CommitAll!"), + A("world")))); + } + + AddVariableDeclarations(); + + State.Statements.push_back(L(A("return"), A("world"))); + + if (DqEngineEnabled) { + State.Statements[dqEnginePgmPos] = L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, + QA("DqEngine"), QA(DqEngineForce ? "force" : "auto"))); } else { - for (int i = 0; i < ListLength(raw); ++i) { - if (!ParseRawStmt(LIST_CAST_NTH(RawStmt, raw, i))) { - return nullptr; - } + State.Statements.erase(State.Statements.begin() + dqEnginePgmPos); + } + + if (State.CostBasedOptimizer) { + State.Statements[costBasedOptimizerPos] = L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, + QA("CostBasedOptimizer"), QA(State.CostBasedOptimizer))); + } else { + State.Statements.erase(State.Statements.begin() + costBasedOptimizerPos); + } + + if (BlockEngineEnabled) { + State.Statements[blockEnginePgmPos] = L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, + QA("BlockEngine"), QA(BlockEngineForce ? "force" : "auto"))); + } else { + State.Statements.erase(State.Statements.begin() + blockEnginePgmPos); + } + + return VL(State.Statements.data(), State.Statements.size()); + } + + TAstNode* ParseResult(const List* raw) { + auto configSource = L(A("DataSource"), QA(TString(NYql::ConfigProviderName))); + State.Statements.push_back(L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, + QA("OrderedColumns")))); + + ui32 blockEnginePgmPos = State.Statements.size(); + State.Statements.push_back(configSource); + ui32 costBasedOptimizerPos = State.Statements.size(); + State.Statements.push_back(configSource); + ui32 dqEnginePgmPos = State.Statements.size(); + State.Statements.push_back(configSource); + + for (int i = 0; i < ListLength(raw); ++i) { + if (!ParseRawStmt(LIST_CAST_NTH(RawStmt, raw, i))) { + return nullptr; } } - if (!Views.empty()) { + if (!State.Views.empty()) { AddError("Not all views have been dropped"); return nullptr; } if (Settings.EndOfQueryCommit) { - Statements.push_back(L(A("let"), A("world"), L(A("CommitAll!"), + State.Statements.push_back(L(A("let"), A("world"), L(A("CommitAll!"), A("world")))); } AddVariableDeclarations(); - Statements.push_back(L(A("return"), A("world"))); + State.Statements.push_back(L(A("return"), A("world"))); if (DqEngineEnabled) { - Statements[dqEnginePgmPos] = L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, + State.Statements[dqEnginePgmPos] = L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, QA("DqEngine"), QA(DqEngineForce ? "force" : "auto"))); } else { - Statements.erase(Statements.begin() + dqEnginePgmPos); + State.Statements.erase(State.Statements.begin() + dqEnginePgmPos); } - if (CostBasedOptimizer) { - Statements[costBasedOptimizerPos] = L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, - QA("CostBasedOptimizer"), QA(CostBasedOptimizer))); + if (State.CostBasedOptimizer) { + State.Statements[costBasedOptimizerPos] = L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, + QA("CostBasedOptimizer"), QA(State.CostBasedOptimizer))); } else { - Statements.erase(Statements.begin() + costBasedOptimizerPos); + State.Statements.erase(State.Statements.begin() + costBasedOptimizerPos); } if (BlockEngineEnabled) { - Statements[blockEnginePgmPos] = L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, + State.Statements[blockEnginePgmPos] = L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, QA("BlockEngine"), QA(BlockEngineForce ? "force" : "auto"))); } else { - Statements.erase(Statements.begin() + blockEnginePgmPos); + State.Statements.erase(State.Statements.begin() + blockEnginePgmPos); } - return VL(Statements.data(), Statements.size()); + return VL(State.Statements.data(), State.Statements.size()); } [[nodiscard]] @@ -560,8 +631,8 @@ class TConverter : public IPGParseEvents { using TAutoParamName = TString; TAutoParamName AddAutoParam(Ydb::TypedValue&& val) { - auto nextName = TString(AUTO_PARAM_PREFIX) + ToString(AutoParamValues.size()); - AutoParamValues.emplace(nextName, std::move(val)); + auto nextName = TString(AUTO_PARAM_PREFIX) + ToString(State.AutoParamValues.size()); + State.AutoParamValues.emplace(nextName, std::move(val)); return nextName; } @@ -588,9 +659,9 @@ class TConverter : public IPGParseEvents { const auto paramType = L(A("ListType"), VL(autoParamTupleType)); const auto paramName = AddAutoParam(MakeYdbListTupleParamValue(std::move(ydbValues), std::move(columnTypes))); - Statements.push_back(L(A("declare"), A(paramName), paramType)); + State.Statements.push_back(L(A("declare"), A(paramName), paramType)); - YQL_CLOG(INFO, Default) << "Successfully autoparametrized VALUES at" << Positions.back(); + YQL_CLOG(INFO, Default) << "Successfully autoparametrized VALUES at" << State.Positions.back(); return A(paramName); } @@ -726,9 +797,9 @@ class TConverter : public IPGParseEvents { ) { bool isValuesClauseOfInsertStmt = fillTargetColumns; - CTE.emplace_back(); + State.CTE.emplace_back(); Y_DEFER { - CTE.pop_back(); + State.CTE.pop_back(); }; if (value->withClause) { @@ -1233,13 +1304,13 @@ class TConverter : public IPGParseEvents { } auto resOptions = QL(QL(QA("type")), QL(QA("autoref"))); - Statements.push_back(L(A("let"), A("output"), output)); - Statements.push_back(L(A("let"), A("result_sink"), L(A("DataSink"), QA(TString(NYql::ResultProviderName))))); - Statements.push_back(L(A("let"), A("world"), L(A("Write!"), + State.Statements.push_back(L(A("let"), A("output"), output)); + State.Statements.push_back(L(A("let"), A("result_sink"), L(A("DataSink"), QA(TString(NYql::ResultProviderName))))); + State.Statements.push_back(L(A("let"), A("world"), L(A("Write!"), A("world"), A("result_sink"), L(A("Key")), A("output"), resOptions))); - Statements.push_back(L(A("let"), A("world"), L(A("Commit!"), + State.Statements.push_back(L(A("let"), A("world"), L(A("Commit!"), A("world"), A("result_sink")))); - return Statements.back(); + return State.Statements.back(); } [[nodiscard]] @@ -1291,7 +1362,7 @@ class TConverter : public IPGParseEvents { return false; } - auto& currentCTEs = CTE.back(); + auto& currentCTEs = State.CTE.back(); if (currentCTEs.find(view.Name) != currentCTEs.end()) { AddError(TStringBuilder() << "CTE already exists: '" << view.Name << "'"); return false; @@ -1444,7 +1515,7 @@ class TConverter : public IPGParseEvents { const auto writeOptions = BuildWriteOptions(value, std::move(returningList)); - Statements.push_back(L( + State.Statements.push_back(L( A("let"), A("world"), L( @@ -1457,7 +1528,7 @@ class TConverter : public IPGParseEvents { ) )); - return Statements.back(); + return State.Statements.back(); } [[nodiscard]] @@ -1516,13 +1587,13 @@ class TConverter : public IPGParseEvents { L(A("Void")), QVL(options.data(), options.size()))) )); - Statements.push_back(L( + State.Statements.push_back(L( A("let"), A("world"), writeUpdate )); - return Statements.back(); + return State.Statements.back(); } [[nodiscard]] @@ -1580,14 +1651,14 @@ class TConverter : public IPGParseEvents { return nullptr; } - auto it = Views.find(view.Name); - if (it != Views.end() && !value->replace) { + auto it = State.Views.find(view.Name); + if (it != State.Views.end() && !value->replace) { AddError(TStringBuilder() << "View already exists: '" << view.Name << "'"); return nullptr; } - Views[view.Name] = view; - return Statements.back(); + State.Views[view.Name] = view; + return State.Statements.back(); } #pragma region CreateTable @@ -1960,12 +2031,12 @@ class TConverter : public IPGParseEvents { } } - Statements.push_back( + State.Statements.push_back( L(A("let"), A("world"), L(A("Write!"), A("world"), sink, key, L(A("Void")), BuildCreateTableOptions(ctx)))); - return Statements.back(); + return State.Statements.back(); } #pragma endregion CreateTable @@ -2014,18 +2085,18 @@ class TConverter : public IPGParseEvents { } const auto name = StrVal(nameNode); - auto it = Views.find(name); - if (!value->missing_ok && it == Views.end()) { + auto it = State.Views.find(name); + if (!value->missing_ok && it == State.Views.end()) { AddError(TStringBuilder() << "View not found: '" << name << "'"); return nullptr; } - if (it != Views.end()) { - Views.erase(it); + if (it != State.Views.end()) { + State.Views.erase(it); } } - return Statements.back(); + return State.Statements.back(); } TAstNode* ParseDropTableStmt(const DropStmt* value, const TVector& names) { @@ -2048,7 +2119,7 @@ class TConverter : public IPGParseEvents { } TString mode = (value->missing_ok) ? "drop_if_exists" : "drop"; - Statements.push_back(L( + State.Statements.push_back(L( A("let"), A("world"), L( @@ -2064,7 +2135,7 @@ class TConverter : public IPGParseEvents { )); } - return Statements.back(); + return State.Statements.back(); } TAstNode* ParseDropIndexStmt(const DropStmt* value, const TVector& names) { @@ -2088,7 +2159,7 @@ class TConverter : public IPGParseEvents { ); TString missingOk = (value->missing_ok) ? "true" : "false"; - Statements.push_back(L( + State.Statements.push_back(L( A("let"), A("world"), L( @@ -2105,7 +2176,7 @@ class TConverter : public IPGParseEvents { )); } - return Statements.back(); + return State.Statements.back(); } [[nodiscard]] @@ -2138,7 +2209,7 @@ class TConverter : public IPGParseEvents { if (Settings.GUCSettings) { Settings.GUCSettings->Set(name, rawStr, value->is_local); } - return Statements.back(); + return State.Statements.back(); } if (name == "useblocks" || name == "emitaggapply") { @@ -2151,7 +2222,7 @@ class TConverter : public IPGParseEvents { if (NodeTag(arg) == T_A_Const && (NodeTag(CAST_NODE(A_Const, arg)->val) == T_String)) { TString rawStr = StrVal(CAST_NODE(A_Const, arg)->val); auto configSource = L(A("DataSource"), QA(TString(NYql::ConfigProviderName))); - Statements.push_back(L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, + State.Statements.push_back(L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), configSource, QA(TString(rawStr == "true" ? "" : "Disable") + TString((name == "useblocks") ? "UseBlocks" : "PgEmitAggApply"))))); } else { AddError(TStringBuilder() << "VariableSetStmt, expected string literal for " << value->name << " option"); @@ -2210,7 +2281,7 @@ class TConverter : public IPGParseEvents { auto rawStr = StrVal(CAST_NODE(A_Const, arg)->val); - Statements.push_back(L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), providerSource, + State.Statements.push_back(L(A("let"), A("world"), L(A(TString(NYql::ConfigureName)), A("world"), providerSource, QA("Attr"), QAX(name.substr(dotPos + 1)), QAX(rawStr)))); } else { AddError(TStringBuilder() << "VariableSetStmt, expected string literal for " << value->name << " option"); @@ -2245,7 +2316,7 @@ class TConverter : public IPGParseEvents { return nullptr; } - CostBasedOptimizer = str; + State.CostBasedOptimizer = str; } else { AddError(TStringBuilder() << "VariableSetStmt, expected string literal for " << value->name << " option"); return nullptr; @@ -2255,7 +2326,7 @@ class TConverter : public IPGParseEvents { return nullptr; } - return Statements.back(); + return State.Statements.back(); } [[nodiscard]] @@ -2334,7 +2405,7 @@ class TConverter : public IPGParseEvents { if (!returningList.empty()) { options.push_back(QL(QA("returning"), QVL(returningList.data(), returningList.size()))); } - Statements.push_back(L( + State.Statements.push_back(L( A("let"), A("world"), L( @@ -2346,7 +2417,7 @@ class TConverter : public IPGParseEvents { QVL(options.data(), options.size()) ) )); - return Statements.back(); + return State.Statements.back(); } TMaybe GetConfigVariable(const TString& varName) { @@ -2405,15 +2476,15 @@ class TConverter : public IPGParseEvents { const auto selectOptions = QL(setItems, setOps); const auto output = L(A("PgSelect"), selectOptions); - Statements.push_back(L(A("let"), A("output"), output)); - Statements.push_back(L(A("let"), A("result_sink"), L(A("DataSink"), QA(TString(NYql::ResultProviderName))))); + State.Statements.push_back(L(A("let"), A("output"), output)); + State.Statements.push_back(L(A("let"), A("result_sink"), L(A("DataSink"), QA(TString(NYql::ResultProviderName))))); const auto resOptions = QL(QL(QA("type")), QL(QA("autoref"))); - Statements.push_back(L(A("let"), A("world"), L(A("Write!"), + State.Statements.push_back(L(A("let"), A("world"), L(A("Write!"), A("world"), A("result_sink"), L(A("Key")), A("output"), resOptions))); - Statements.push_back(L(A("let"), A("world"), L(A("Commit!"), + State.Statements.push_back(L(A("let"), A("world"), L(A("Commit!"), A("world"), A("result_sink")))); - return Statements.back(); + return State.Statements.back(); } [[nodiscard]] @@ -2423,14 +2494,14 @@ class TConverter : public IPGParseEvents { case TRANS_STMT_START: return true; case TRANS_STMT_COMMIT: - Statements.push_back(L(A("let"), A("world"), L(A("CommitAll!"), + State.Statements.push_back(L(A("let"), A("world"), L(A("CommitAll!"), A("world")))); if (Settings.GUCSettings) { Settings.GUCSettings->Commit(); } return true; case TRANS_STMT_ROLLBACK: - Statements.push_back(L(A("let"), A("world"), L(A("CommitAll!"), + State.Statements.push_back(L(A("let"), A("world"), L(A("CommitAll!"), A("world"), QL(QL(QA("mode"), QA("rollback")))))); if (Settings.GUCSettings) { Settings.GUCSettings->RollBack(); @@ -2500,7 +2571,7 @@ class TConverter : public IPGParseEvents { desc.emplace_back(QL(QA("dataColumns"), QVL(coverColumns->data(), coverColumns->size()))); desc.emplace_back(QL(QA("flags"), QVL(flags.data(), flags.size()))); - Statements.push_back(L( + State.Statements.push_back(L( A("let"), A("world"), L( @@ -2516,7 +2587,7 @@ class TConverter : public IPGParseEvents { ) )); - return Statements.back(); + return State.Statements.back(); } TFromDesc ParseFromClause(const Node* node) { @@ -2542,11 +2613,11 @@ class TConverter : public IPGParseEvents { auto colNamesTuple = QVL(colNamesNodes.data(), colNamesNodes.size()); if (p.InjectRead) { - auto label = "read" + ToString(ReadIndex); - Statements.push_back(L(A("let"), A(label), p.Source)); - Statements.push_back(L(A("let"), A("world"), L(A("Left!"), A(label)))); + auto label = "read" + ToString(State.ReadIndex); + State.Statements.push_back(L(A("let"), A(label), p.Source)); + State.Statements.push_back(L(A("let"), A("world"), L(A("Left!"), A(label)))); fromList.push_back(QL(L(A("Right!"), A(label)), aliasNode, colNamesTuple)); - ++ReadIndex; + ++State.ReadIndex; } else { fromList.push_back(QL(p.Source, aliasNode, colNamesTuple)); } @@ -2671,7 +2742,7 @@ class TConverter : public IPGParseEvents { const TView* view = nullptr; if (StrLength(value->schemaname) == 0) { - for (auto rit = CTE.rbegin(); rit != CTE.rend(); ++rit) { + for (auto rit = State.CTE.rbegin(); rit != State.CTE.rend(); ++rit) { auto cteIt = rit->find(value->relname); if (cteIt != rit->end()) { view = &cteIt->second; @@ -2679,8 +2750,8 @@ class TConverter : public IPGParseEvents { } } if (!view) { - auto viewIt = Views.find(value->relname); - if (viewIt != Views.end()) { + auto viewIt = State.Views.find(value->relname); + if (viewIt != State.Views.end()) { view = &viewIt->second; } } @@ -2991,8 +3062,8 @@ class TConverter : public IPGParseEvents { TAstNode* ParseParamRefExpr(const ParamRef* value) { const auto varName = PREPARED_PARAM_PREFIX + ToString(value->number); - if (!ParamNameToPgTypeName.contains(varName)) { - ParamNameToPgTypeName[varName] = DEFAULT_PARAM_TYPE; + if (!State.ParamNameToPgTypeName.contains(varName)) { + State.ParamNameToPgTypeName[varName] = DEFAULT_PARAM_TYPE; } return A(varName); } @@ -3132,9 +3203,9 @@ class TConverter : public IPGParseEvents { } const auto& paramName = AddAutoParam(std::move(typedValue)); - Statements.push_back(L(A("declare"), A(paramName), pgType)); + State.Statements.push_back(L(A("declare"), A(paramName), pgType)); - YQL_CLOG(INFO, Default) << "Autoparametrized " << paramName << " at " << Positions.back(); + YQL_CLOG(INFO, Default) << "Autoparametrized " << paramName << " at " << State.Positions.back(); return A(paramName); } @@ -4244,9 +4315,9 @@ class TConverter : public IPGParseEvents { } void AddVariableDeclarations() { - for (const auto& [varName, typeName] : ParamNameToPgTypeName) { + for (const auto& [varName, typeName] : State.ParamNameToPgTypeName) { const auto pgType = L(A("PgType"), QA(typeName)); - Statements.push_back(L(A("declare"), A(varName), pgType)); + State.Statements.push_back(L(A("declare"), A(varName), pgType)); } } @@ -4285,11 +4356,11 @@ class TConverter : public IPGParseEvents { } TAstNode* VL(TAstNode** nodes, ui32 size, TPosition pos = {}) { - return TAstNode::NewList(pos.Row ? pos : Positions.back(), nodes, size, *AstParseResult.Pool); + return TAstNode::NewList(pos.Row ? pos : State.Positions.back(), nodes, size, *AstParseResults[StatementId].Pool); } TAstNode* VL(TArrayRef nodes, TPosition pos = {}) { - return TAstNode::NewList(pos.Row ? pos : Positions.back(), nodes.data(), nodes.size(), *AstParseResult.Pool); + return TAstNode::NewList(pos.Row ? pos : State.Positions.back(), nodes.data(), nodes.size(), *AstParseResults[StatementId].Pool); } TAstNode* QVL(TAstNode** nodes, ui32 size, TPosition pos = {}) { @@ -4305,11 +4376,11 @@ class TConverter : public IPGParseEvents { } TAstNode* A(const TStringBuf str, TPosition pos = {}, ui32 flags = 0) { - return TAstNode::NewAtom(pos.Row ? pos : Positions.back(), str, *AstParseResult.Pool, flags); + return TAstNode::NewAtom(pos.Row ? pos : State.Positions.back(), str, *AstParseResults[StatementId].Pool, flags); } TAstNode* AX(const TString& str, TPosition pos = {}) { - return A(str, pos.Row ? pos : Positions.back(), TNodeFlags::ArbitraryContent); + return A(str, pos.Row ? pos : State.Positions.back(), TNodeFlags::ArbitraryContent); } TAstNode* Q(TAstNode* node, TPosition pos = {}) { @@ -4328,7 +4399,7 @@ class TConverter : public IPGParseEvents { TAstNode* L(TNodes... nodes) { TLState state; LImpl(state, nodes...); - return TAstNode::NewList(state.Position.Row ? state.Position : Positions.back(), state.Nodes.data(), state.Nodes.size(), *AstParseResult.Pool); + return TAstNode::NewList(state.Position.Row ? state.Position : State.Positions.back(), state.Nodes.data(), state.Nodes.size(), *AstParseResults[StatementId].Pool); } template @@ -4352,11 +4423,11 @@ class TConverter : public IPGParseEvents { private: void AddError(const TString& value) { - AstParseResult.Issues.AddIssue(TIssue(Positions.back(), value)); + AstParseResults[StatementId].Issues.AddIssue(TIssue(State.Positions.back(), value)); } void AddWarning(int code, const TString& value) { - AstParseResult.Issues.AddIssue(TIssue(Positions.back(), value).SetCode(code, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + AstParseResults[StatementId].Issues.AddIssue(TIssue(State.Positions.back(), value).SetCode(code, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); } struct TLState { @@ -4387,15 +4458,15 @@ class TConverter : public IPGParseEvents { void PushPosition(int location) { if (location == -1) { - Positions.push_back(Positions.back()); + State.Positions.push_back(State.Positions.back()); return; } - Positions.push_back(Location2Position(location)); + State.Positions.push_back(Location2Position(location)); }; void PopPosition() { - Positions.pop_back(); + State.Positions.pop_back(); } NYql::TPosition Location2Position(int location) const { @@ -4447,26 +4518,20 @@ class TConverter : public IPGParseEvents { } private: - TAstParseResult& AstParseResult; + TVector& AstParseResults; NSQLTranslation::TTranslationSettings Settings; bool DqEngineEnabled = false; bool DqEngineForce = false; - TString CostBasedOptimizer; bool BlockEngineEnabled = false; bool BlockEngineForce = false; - TVector Statements; - ui32 ReadIndex = 0; - TViews Views; - TVector CTE; TString TablePathPrefix; - TVector Positions; TVector RowStarts; ui32 QuerySize; TString Provider; static const THashMap ProviderToInsertModeMap; - THashMap ParamNameToPgTypeName; - THashMap AutoParamValues; + TState State; + ui32 StatementId = 0; }; const THashMap TConverter::ProviderToInsertModeMap = { @@ -4475,25 +4540,17 @@ const THashMap TConverter::ProviderToInsertModeMap = { }; NYql::TAstParseResult PGToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings) { - NYql::TAstParseResult result; - TConverter converter(result, settings, query); + TVector results; + TConverter converter(results, settings, query); NYql::PGParse(query, converter); - return std::move(result); + return std::move(results.back()); } TVector PGToYqlStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings) { TVector results; - List* raw = NYql::PGGetStatements(query); - if (!raw) { - return {}; - } - for (int statementId = 0; statementId < ListLength(raw); ++statementId) { - NYql::TAstParseResult result; - TConverter converter(result, settings, query); - converter.OnResult(raw, statementId); - results.push_back(std::move(result)); - } - return results; + TConverter converter(results, settings, query); + NYql::PGParse(query, converter); + return std::move(results); } } // NSQLTranslationPG From e48ae0eb769f1058346e819b9f4ca55b0d46a94a Mon Sep 17 00:00:00 2001 From: Polina Volosnikova Date: Thu, 15 Feb 2024 12:38:17 +0000 Subject: [PATCH 5/5] delete extra --- ydb/core/kqp/session_actor/kqp_session_actor.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index 197c13375603..90be067875d4 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -694,9 +694,6 @@ class TKqpSessionActor : public TActorBootstrapped { AppData()->TimeProvider, AppData()->RandomProvider, Config->EnableKqpImmediateEffects); QueryState->QueryData = std::make_shared(QueryState->TxCtx->TxAlloc); QueryState->TxCtx->EffectiveIsolationLevel = NKikimrKqp::ISOLATION_LEVEL_UNDEFINED; - Ydb::Table::TransactionSettings settings; - settings.mutable_serializable_read_write(); - BeginTx(settings); } return true;