From 891391b78cc1d75f20660f87ad47cc2032ac55df Mon Sep 17 00:00:00 2001 From: Daniil Demin Date: Tue, 6 Aug 2024 12:20:51 +0000 Subject: [PATCH] Minimal change to enable aggregates inside VIEW queries --- ydb/core/kqp/provider/rewrite_io_utils.cpp | 10 ++-- ydb/core/kqp/provider/rewrite_io_utils.h | 3 +- .../kqp/provider/yql_kikimr_datasource.cpp | 3 +- .../aggregates_and_window/create_view.sql | 43 +++++++++++++++++ .../cases/aggregates_and_window/drop_view.sql | 1 + .../aggregates_and_window/etalon_query.sql | 46 +++++++++++++++++++ .../select_from_view.sql | 3 ++ .../cases/count_episodes/create_view.sql | 9 ++++ .../input/cases/count_episodes/drop_view.sql | 1 + .../cases/count_episodes/etalon_query.sql | 12 +++++ .../cases/count_episodes/select_from_view.sql | 3 ++ .../create_view.sql | 22 +++++++++ .../count_episodes_with_titles/drop_view.sql | 1 + .../etalon_query.sql | 25 ++++++++++ .../select_from_view.sql | 3 ++ .../input/cases/count_rows/create_view.sql | 4 ++ .../view/input/cases/count_rows/drop_view.sql | 1 + .../input/cases/count_rows/etalon_query.sql | 7 +++ .../cases/count_rows/select_from_view.sql | 3 ++ 19 files changed, 194 insertions(+), 6 deletions(-) create mode 100644 ydb/core/kqp/ut/view/input/cases/aggregates_and_window/create_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/aggregates_and_window/drop_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/aggregates_and_window/etalon_query.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/aggregates_and_window/select_from_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_episodes/create_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_episodes/drop_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_episodes/etalon_query.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_episodes/select_from_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/create_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/drop_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/etalon_query.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/select_from_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_rows/create_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_rows/drop_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_rows/etalon_query.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/count_rows/select_from_view.sql diff --git a/ydb/core/kqp/provider/rewrite_io_utils.cpp b/ydb/core/kqp/provider/rewrite_io_utils.cpp index e9262bd91aaf..85226e438ca2 100644 --- a/ydb/core/kqp/provider/rewrite_io_utils.cpp +++ b/ydb/core/kqp/provider/rewrite_io_utils.cpp @@ -28,7 +28,8 @@ NSQLTranslation::TTranslationSettings CreateViewTranslationSettings(const TStrin TExprNode::TPtr CompileViewQuery( const TString& query, TExprContext& ctx, - const TString& cluster + const TString& cluster, + IModuleResolver::TPtr moduleResolver ) { TAstParseResult queryAst; queryAst = NSQLTranslation::SqlToYql(query, CreateViewTranslationSettings(cluster)); @@ -39,7 +40,7 @@ TExprNode::TPtr CompileViewQuery( } TExprNode::TPtr queryGraph; - if (!CompileExpr(*queryAst.Root, queryGraph, ctx, nullptr, nullptr)) { + if (!CompileExpr(*queryAst.Root, queryGraph, ctx, moduleResolver.get(), nullptr)) { return nullptr; } @@ -123,14 +124,15 @@ TExprNode::TPtr RewriteReadFromView( const TExprNode::TPtr& node, TExprContext& ctx, const TString& query, - const TString& cluster + const TString& cluster, + IModuleResolver::TPtr moduleResolver ) { const TCoRead readNode(node->ChildPtr(0)); const auto worldBeforeThisRead = readNode.World().Ptr(); TExprNode::TPtr queryGraph = FindSavedQueryGraph(readNode.Ptr()); if (!queryGraph) { - queryGraph = CompileViewQuery(query, ctx, cluster); + queryGraph = CompileViewQuery(query, ctx, cluster, moduleResolver); if (!queryGraph) { ctx.AddError(TIssue(ctx.GetPosition(readNode.Pos()), "The query stored in the view cannot be compiled.")); diff --git a/ydb/core/kqp/provider/rewrite_io_utils.h b/ydb/core/kqp/provider/rewrite_io_utils.h index 02bcdfc514cf..3b18be660c94 100644 --- a/ydb/core/kqp/provider/rewrite_io_utils.h +++ b/ydb/core/kqp/provider/rewrite_io_utils.h @@ -8,7 +8,8 @@ TExprNode::TPtr RewriteReadFromView( const TExprNode::TPtr& node, TExprContext& ctx, const TString& query, - const TString& cluster + const TString& cluster, + IModuleResolver::TPtr moduleResolver ); } \ No newline at end of file diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp index af95adf40d5a..59a485d8c51c 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp @@ -754,6 +754,7 @@ class TKikimrDataSource : public TDataProviderBase { } ctx.Step + .Repeat(TExprStep::ExpandApplyForLambdas) .Repeat(TExprStep::ExprEval) .Repeat(TExprStep::DiscoveryIO) .Repeat(TExprStep::Epochs) @@ -762,7 +763,7 @@ class TKikimrDataSource : public TDataProviderBase { .Repeat(TExprStep::RewriteIO); const auto& query = tableDesc.Metadata->ViewPersistedData.QueryText; - return RewriteReadFromView(node, ctx, query, cluster); + return RewriteReadFromView(node, ctx, query, cluster, Types.Modules); } } diff --git a/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/create_view.sql b/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/create_view.sql new file mode 100644 index 000000000000..f9f813948191 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/create_view.sql @@ -0,0 +1,43 @@ +CREATE VIEW `/Root/aggregates_and_window` WITH (security_invoker = TRUE) AS + SELECT + series.title AS series, + series_stats.seasons_with_episode_count_greater_than_average AS seasons_with_episode_count_greater_than_average + FROM ( + SELECT + series_id, + SUM( + CASE + WHEN episode_count > average_episodes_in_season + THEN 1 + ELSE 0 + END + ) AS seasons_with_episode_count_greater_than_average + FROM ( + SELECT + series_id, + season_id, + episode_count, + AVG(episode_count) OVER average_episodes_in_season_window AS average_episodes_in_season + FROM ( + SELECT + series_id, + season_id, + COUNT(*) AS episode_count + FROM `/Root/episodes` + GROUP BY + series_id, + season_id + ) + WINDOW + average_episodes_in_season_window AS ( + PARTITION BY + series_id + ) + ) + GROUP BY + series_id + ) + AS series_stats + JOIN `/Root/series` + AS series + USING (series_id); diff --git a/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/drop_view.sql b/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/drop_view.sql new file mode 100644 index 000000000000..c4c0dd0a1c8f --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/drop_view.sql @@ -0,0 +1 @@ +DROP VIEW `/Root/aggregates_and_window`; diff --git a/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/etalon_query.sql b/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/etalon_query.sql new file mode 100644 index 000000000000..c701bd2d2648 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/etalon_query.sql @@ -0,0 +1,46 @@ +SELECT + * +FROM ( + SELECT + series.title AS series, + series_stats.seasons_with_episode_count_greater_than_average AS seasons_with_episode_count_greater_than_average + FROM ( + SELECT + series_id, + SUM( + CASE + WHEN episode_count > average_episodes_in_season + THEN 1 + ELSE 0 + END + ) AS seasons_with_episode_count_greater_than_average + FROM ( + SELECT + series_id, + season_id, + episode_count, + AVG(episode_count) OVER average_episodes_in_season_window AS average_episodes_in_season + FROM ( + SELECT + series_id, + season_id, + COUNT(*) AS episode_count + FROM `/Root/episodes` + GROUP BY + series_id, + season_id + ) + WINDOW + average_episodes_in_season_window AS ( + PARTITION BY + series_id + ) + ) + GROUP BY + series_id + ) + AS series_stats + JOIN `/Root/series` + AS series + USING (series_id) +); diff --git a/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/select_from_view.sql b/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/select_from_view.sql new file mode 100644 index 000000000000..0cc947dd8661 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/aggregates_and_window/select_from_view.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `/Root/aggregates_and_window`; diff --git a/ydb/core/kqp/ut/view/input/cases/count_episodes/create_view.sql b/ydb/core/kqp/ut/view/input/cases/count_episodes/create_view.sql new file mode 100644 index 000000000000..dcbfee8f92db --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_episodes/create_view.sql @@ -0,0 +1,9 @@ +CREATE VIEW `/Root/count_episodes` WITH (security_invoker = TRUE) AS + SELECT + series_id, + season_id, + COUNT(*) + FROM `/Root/episodes` + GROUP BY + series_id, + season_id; diff --git a/ydb/core/kqp/ut/view/input/cases/count_episodes/drop_view.sql b/ydb/core/kqp/ut/view/input/cases/count_episodes/drop_view.sql new file mode 100644 index 000000000000..897adef6e6d6 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_episodes/drop_view.sql @@ -0,0 +1 @@ +DROP VIEW `/Root/count_episodes`; diff --git a/ydb/core/kqp/ut/view/input/cases/count_episodes/etalon_query.sql b/ydb/core/kqp/ut/view/input/cases/count_episodes/etalon_query.sql new file mode 100644 index 000000000000..8964c9ab76a1 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_episodes/etalon_query.sql @@ -0,0 +1,12 @@ +SELECT + * +FROM ( + SELECT + series_id, + season_id, + COUNT(*) + FROM `/Root/episodes` + GROUP BY + series_id, + season_id +); diff --git a/ydb/core/kqp/ut/view/input/cases/count_episodes/select_from_view.sql b/ydb/core/kqp/ut/view/input/cases/count_episodes/select_from_view.sql new file mode 100644 index 000000000000..153ad494f165 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_episodes/select_from_view.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `/Root/count_episodes`; diff --git a/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/create_view.sql b/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/create_view.sql new file mode 100644 index 000000000000..545ec4056034 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/create_view.sql @@ -0,0 +1,22 @@ +CREATE VIEW `/Root/count_episodes_with_titles` WITH (security_invoker = TRUE) AS + SELECT + series.title AS series, + seasons.title AS season, + episodes.episode_count AS episode_count + FROM ( + SELECT + series_id, + season_id, + COUNT(*) AS episode_count + FROM `/Root/episodes` + GROUP BY + series_id, + season_id + ) + AS episodes + JOIN `/Root/series` + AS series + ON episodes.series_id == series.series_id + JOIN `/Root/seasons` + AS seasons + ON episodes.series_id == seasons.series_id AND episodes.season_id == seasons.season_id; diff --git a/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/drop_view.sql b/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/drop_view.sql new file mode 100644 index 000000000000..99e8b54a675d --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/drop_view.sql @@ -0,0 +1 @@ +DROP VIEW `/Root/count_episodes_with_titles`; diff --git a/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/etalon_query.sql b/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/etalon_query.sql new file mode 100644 index 000000000000..faa472a66beb --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/etalon_query.sql @@ -0,0 +1,25 @@ +SELECT + * +FROM ( + SELECT + series.title AS series, + seasons.title AS season, + episodes.episode_count AS episode_count + FROM ( + SELECT + series_id, + season_id, + COUNT(*) AS episode_count + FROM `/Root/episodes` + GROUP BY + series_id, + season_id + ) + AS episodes + JOIN `/Root/series` + AS series + ON episodes.series_id == series.series_id + JOIN `/Root/seasons` + AS seasons + ON episodes.series_id == seasons.series_id AND episodes.season_id == seasons.season_id +); diff --git a/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/select_from_view.sql b/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/select_from_view.sql new file mode 100644 index 000000000000..b84299a26d02 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_episodes_with_titles/select_from_view.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `/Root/count_episodes_with_titles`; diff --git a/ydb/core/kqp/ut/view/input/cases/count_rows/create_view.sql b/ydb/core/kqp/ut/view/input/cases/count_rows/create_view.sql new file mode 100644 index 000000000000..0189412ace53 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_rows/create_view.sql @@ -0,0 +1,4 @@ +CREATE VIEW `/Root/count_rows` WITH (security_invoker = TRUE) AS + SELECT + COUNT(*) + FROM `/Root/episodes`; diff --git a/ydb/core/kqp/ut/view/input/cases/count_rows/drop_view.sql b/ydb/core/kqp/ut/view/input/cases/count_rows/drop_view.sql new file mode 100644 index 000000000000..b540f8f401e8 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_rows/drop_view.sql @@ -0,0 +1 @@ +DROP VIEW `/Root/count_rows`; diff --git a/ydb/core/kqp/ut/view/input/cases/count_rows/etalon_query.sql b/ydb/core/kqp/ut/view/input/cases/count_rows/etalon_query.sql new file mode 100644 index 000000000000..b9d2cdb0f4a7 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_rows/etalon_query.sql @@ -0,0 +1,7 @@ +SELECT + * +FROM ( + SELECT + COUNT(*) + FROM `/Root/episodes` +); diff --git a/ydb/core/kqp/ut/view/input/cases/count_rows/select_from_view.sql b/ydb/core/kqp/ut/view/input/cases/count_rows/select_from_view.sql new file mode 100644 index 000000000000..49ffdf0dd923 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/count_rows/select_from_view.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `/Root/count_rows`;