Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support of nulls first/last #2365

Merged
merged 3 commits into from
Mar 1, 2024
Merged

Support of nulls first/last #2365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0176498
init
vitstn Feb 29, 2024
c6a54c0
tests
vitstn Mar 1, 2024
ae582c5
pg compat switch
vitstn Mar 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 56 additions & 51 deletions ydb/library/yql/core/common_opt/yql_co_pgselect.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2262,66 +2262,57 @@ std::tuple<TExprNode::TPtr, TExprNode::TPtr> BuildFrame(TPositionHandle pos, con

TExprNode::TPtr BuildSortTraits(TPositionHandle pos, const TExprNode& sortColumns, const TExprNode::TPtr& list,
const TAggregationMap* aggId, TExprContext& ctx, TOptimizeContext& optCtx) {
if (sortColumns.ChildrenSize() == 1) {
auto lambda = sortColumns.Head().ChildPtr(1);
if (aggId) {
RewriteAggs(lambda, *aggId, ctx, optCtx, false);
}

return ctx.Builder(pos)
.Callable("SortTraits")
.Callable(0, "TypeOf")
.Add(0, list)
.Seal()
.Callable(1, "Bool")
.Atom(0, sortColumns.Head().Tail().Content() == "asc" ? "true" : "false")
.Seal()
.Lambda(2)
.Param("row")
.Apply(lambda)
.With(0, "row")
.Seal()
.Seal()
return ctx.Builder(pos)
.Callable("SortTraits")
.Callable(0, "TypeOf")
.Add(0, list)
.Seal()
.Build();
} else {
return ctx.Builder(pos)
.Callable("SortTraits")
.Callable(0, "TypeOf")
.Add(0, list)
.Seal()
.List(1)
.List(1)
.Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
for (ui32 i = 0; i < sortColumns.ChildrenSize(); ++i) {
parent.Callable(i, "Bool")
.Atom(0, sortColumns.Child(i)->Child(2)->Content() == "asc" ? "true" : "false")
.Seal();
}
return parent;
})
.Seal()
.Lambda(2)
.Param("row")
.List()
.Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
for (ui32 i = 0; i < sortColumns.ChildrenSize(); ++i) {
parent.Callable(i, "Bool")
.Atom(0, sortColumns.Child(i)->Tail().Content() == "asc" ? "true" : "false")
.Seal();
}
return parent;
})
.Seal()
.Lambda(2)
.Param("row")
.List()
.Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
for (ui32 i = 0; i < sortColumns.ChildrenSize(); ++i) {
auto lambda = sortColumns.Child(i)->ChildPtr(1);
if (aggId) {
RewriteAggs(lambda, *aggId, ctx, optCtx, false);
}
auto lambda = sortColumns.Child(i)->ChildPtr(1);
if (aggId) {
RewriteAggs(lambda, *aggId, ctx, optCtx, false);
}

if (sortColumns.Child(i)->ChildPtr(3)->Content() == "last") {
parent.List(i)
.Callable(0, "Not")
.Callable(0, "Exists")
.Apply(0, lambda)
.With(0, "row")
.Seal()
.Seal()
.Seal()
.Apply(1, lambda)
.With(0, "row")
.Seal()
.Seal();
} else {
parent.Apply(i, lambda)
.With(0, "row")
.Seal();
}
}

return parent;
})
.Seal()
return parent;
})
.Seal()
.Seal()
.Build();
}
.Seal()
.Build();
}

TExprNode::TPtr BuildWindows(TPositionHandle pos, const TExprNode::TPtr& list, const TExprNode::TPtr& window, const TWindowsCtx& winCtx,
Expand Down Expand Up @@ -2603,7 +2594,21 @@ TExprNode::TPtr BuildSortLambda(TPositionHandle pos, const TExprNode::TPtr& sort
const auto& keys = sort->Tail();
for (const auto& key : keys.Children()) {
auto keyLambda = key->ChildPtr(1);
rootItems.push_back(ctx.ReplaceNode(keyLambda->TailPtr(), keyLambda->Head().Head(), argNode));
auto value = ctx.ReplaceNode(keyLambda->TailPtr(), keyLambda->Head().Head(), argNode);
if (key->Child(3)->Content() == "last") {
rootItems.push_back(ctx.Builder(pos)
.List()
.Callable(0, "Not")
.Callable(0, "Exists")
.Add(0, value)
.Seal()
.Seal()
.Add(1, value)
.Seal()
.Build());
} else {
rootItems.push_back(value);
}
}

auto root = ctx.NewList(pos, std::move(rootItems));
Expand All @@ -2619,7 +2624,7 @@ TExprNode::TPtr BuildSort(TPositionHandle pos, const TExprNode::TPtr& sort, cons
for (const auto& key : keys.Children()) {
dirItems.push_back(ctx.Builder(pos)
.Callable("Bool")
.Atom(0, key->Tail().Content() == "asc" ? "true" : "false")
.Atom(0, key->Child(2)->Content() == "asc" ? "true" : "false")
.Seal()
.Build());
}
Expand Down
12 changes: 11 additions & 1 deletion ydb/library/yql/core/type_ann/type_ann_pg.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -994,7 +994,7 @@ IGraphTransformer::TStatus PgWhereWrapper(const TExprNode::TPtr& input, TExprNod

IGraphTransformer::TStatus PgSortWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
if (!EnsureArgsCount(*input, 3, ctx.Expr)) {
if (!EnsureArgsCount(*input, 4, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}

Expand All @@ -1008,6 +1008,16 @@ IGraphTransformer::TStatus PgSortWrapper(const TExprNode::TPtr& input, TExprNode
return IGraphTransformer::TStatus::Error;
}

if (!EnsureAtom(*input->Child(3), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}

if (input->Child(3)->Content() != "first" && input->Child(3)->Content() != "last") {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(3)->Pos()),
TStringBuilder() << "Unsupported nulls ordering: " << input->Child(3)->Content()));
return IGraphTransformer::TStatus::Error;
}

bool hasType = false;
if (!input->Child(0)->IsCallable("Void")) {
hasType = true;
Expand Down
20 changes: 16 additions & 4 deletions ydb/library/yql/sql/pg/pg_sql.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4009,10 +4009,13 @@ class TConverter : public IPGParseEvents {
TAstNode* ParseSortBy(const PG_SortBy* value, bool allowAggregates, bool useProjectionRefs) {
AT_LOCATION(value);
bool asc = true;
bool nullsFirst = true;
switch (value->sortby_dir) {
case SORTBY_DEFAULT:
break;
case SORTBY_ASC:
if (Settings.PgSortNulls) {
nullsFirst = false;
}
break;
case SORTBY_DESC:
asc = false;
Expand All @@ -4022,8 +4025,17 @@ class TConverter : public IPGParseEvents {
return nullptr;
}

if (value->sortby_nulls != SORTBY_NULLS_DEFAULT) {
AddError(TStringBuilder() << "sortby_nulls unsupported value: " << (int)value->sortby_nulls);
switch (value->sortby_nulls) {
case SORTBY_NULLS_DEFAULT:
break;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't we want to preserve Postgres default behaviour here? "NULLS FIRST is the default for DESC order, and NULLS LAST otherwise."

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope. It's a breaking change

case SORTBY_NULLS_FIRST:
nullsFirst = true;
break;
case SORTBY_NULLS_LAST:
nullsFirst = false;
break;
default:
AddError(TStringBuilder() << "sortby_dir unsupported value: " << (int)value->sortby_dir);
return nullptr;
}

Expand All @@ -4049,7 +4061,7 @@ class TConverter : public IPGParseEvents {
}

auto lambda = L(A("lambda"), QL(), expr);
return L(A("PgSort"), L(A("Void")), lambda, QA(asc ? "asc" : "desc"));
return L(A("PgSort"), L(A("Void")), lambda, QA(asc ? "asc" : "desc"), QA(nullsFirst ? "first" : "last"));
}

TAstNode* ParseColumnRef(const ColumnRef* value, const TExprSettings& settings) {
Expand Down
1 change: 1 addition & 0 deletions ydb/library/yql/sql/settings/translation_settings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ namespace NSQLTranslation {
bool UnicodeLiterals = false;

TMaybe<TString> ApplicationName;
bool PgSortNulls = false;
};

bool ParseTranslationSettings(const TString& query, NSQLTranslation::TTranslationSettings& settings, NYql::TIssues& issues);
Expand Down
18 changes: 9 additions & 9 deletions ydb/library/yql/tests/sql/dq_file/part0/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2138,9 +2138,9 @@
],
"test.test[pg-pg_types_window2-default.txt-Debug]": [
{
"checksum": "fdb6570b7f0e2e9f698f34e611afdd83",
"size": 1138,
"uri": "https://{canondata_backend}/1924537/c2e51d9e2e0a5d416aad46b25a797e584ed51fa2/resource.tar.gz#test.test_pg-pg_types_window2-default.txt-Debug_/opt.yql_patched"
"checksum": "bc5b35c42c427a5c7e1d0115198b7288",
"size": 1184,
"uri": "https://{canondata_backend}/1937027/751768eb2f05a82d5c0ae53923fca610307cec52/resource.tar.gz#test.test_pg-pg_types_window2-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg-pg_types_window2-default.txt-Plan]": [
Expand Down Expand Up @@ -2314,9 +2314,9 @@
],
"test.test[pg-select_win_partition_sort-default.txt-Debug]": [
{
"checksum": "2aeb060d1f8ab384fc0fc6b2498e9109",
"size": 1408,
"uri": "https://{canondata_backend}/1924537/c2e51d9e2e0a5d416aad46b25a797e584ed51fa2/resource.tar.gz#test.test_pg-select_win_partition_sort-default.txt-Debug_/opt.yql_patched"
"checksum": "d65a3d3d9e2d71503114935166a0fc7e",
"size": 1453,
"uri": "https://{canondata_backend}/1937027/751768eb2f05a82d5c0ae53923fca610307cec52/resource.tar.gz#test.test_pg-select_win_partition_sort-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg-select_win_partition_sort-default.txt-Plan]": [
Expand All @@ -2336,9 +2336,9 @@
],
"test.test[pg-select_win_sum_null-default.txt-Debug]": [
{
"checksum": "fe1b96ce1e0eb7909825980ae27cb0c9",
"size": 1731,
"uri": "https://{canondata_backend}/1924537/c2e51d9e2e0a5d416aad46b25a797e584ed51fa2/resource.tar.gz#test.test_pg-select_win_sum_null-default.txt-Debug_/opt.yql_patched"
"checksum": "4e02da8b8382807b54779ed396e7abf3",
"size": 1737,
"uri": "https://{canondata_backend}/1937027/751768eb2f05a82d5c0ae53923fca610307cec52/resource.tar.gz#test.test_pg-select_win_sum_null-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg-select_win_sum_null-default.txt-Plan]": [
Expand Down
6 changes: 3 additions & 3 deletions ydb/library/yql/tests/sql/dq_file/part1/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2227,9 +2227,9 @@
],
"test.test[pg-tpcds-q86-default.txt-Debug]": [
{
"checksum": "378c7b17aeef13ff3a4632a977d1af05",
"size": 2958,
"uri": "https://{canondata_backend}/995452/b482ddbdc16c2fe4b316d7b636dcbf8ccbf48261/resource.tar.gz#test.test_pg-tpcds-q86-default.txt-Debug_/opt.yql_patched"
"checksum": "22c5b02c719f59d9c2fc97b69660d998",
"size": 2964,
"uri": "https://{canondata_backend}/1936842/e5a3802e20877f63849313081c934fc8a817ce8f/resource.tar.gz#test.test_pg-tpcds-q86-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg-tpcds-q86-default.txt-Plan]": [
Expand Down
12 changes: 6 additions & 6 deletions ydb/library/yql/tests/sql/dq_file/part10/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1948,9 +1948,9 @@
],
"test.test[pg-select_win_min-default.txt-Debug]": [
{
"checksum": "d23f98d8421d7074f2d023ec3ca232a4",
"size": 1601,
"uri": "https://{canondata_backend}/1600758/c778849b827ee6b7243d05d28b4a3f467d62ba71/resource.tar.gz#test.test_pg-select_win_min-default.txt-Debug_/opt.yql_patched"
"checksum": "af0d6edf18a3dbd8b9041fa0d2773fd6",
"size": 1607,
"uri": "https://{canondata_backend}/937458/ef7f65979cf6294805dba5c60647105228b6ef3a/resource.tar.gz#test.test_pg-select_win_min-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg-select_win_min-default.txt-Plan]": [
Expand Down Expand Up @@ -2080,9 +2080,9 @@
],
"test.test[pg-tpcds-q49-default.txt-Debug]": [
{
"checksum": "bf85732f3991115d970f254c9c1b9757",
"size": 6348,
"uri": "https://{canondata_backend}/1600758/32cfdeb8c6377a2e7e62c6c4adbb95f25af7669b/resource.tar.gz#test.test_pg-tpcds-q49-default.txt-Debug_/opt.yql_patched"
"checksum": "b91fb9cd38d6021040be399c5e8d91ae",
"size": 6385,
"uri": "https://{canondata_backend}/1936842/aecf4970df1ec06496312636476de0e7b19c3ebc/resource.tar.gz#test.test_pg-tpcds-q49-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg-tpcds-q49-default.txt-Plan]": [
Expand Down
6 changes: 3 additions & 3 deletions ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1869,9 +1869,9 @@
],
"test.test[pg-select_win_column_order_by-default.txt-Debug]": [
{
"checksum": "5071ebf7d4425b2a67fb877f1e5c1047",
"size": 933,
"uri": "https://{canondata_backend}/1936947/a99026e839b7e22714c2a9a81971a3b5e3ed1eb4/resource.tar.gz#test.test_pg-select_win_column_order_by-default.txt-Debug_/opt.yql_patched"
"checksum": "9c48cc7a72c2ba37bf6c318b0064fe3b",
"size": 939,
"uri": "https://{canondata_backend}/1031349/dcbc10f65e2e4437cc3d26e1817d701c654e06da/resource.tar.gz#test.test_pg-select_win_column_order_by-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg-select_win_column_order_by-default.txt-Plan]": [
Expand Down
6 changes: 3 additions & 3 deletions ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2317,9 +2317,9 @@
],
"test.test[pg-select_win_first_last_value-default.txt-Debug]": [
{
"checksum": "f5557a98fa0aa487964adc592cab8cc0",
"size": 1335,
"uri": "https://{canondata_backend}/1871002/e9746e2cfbb706bb72321a7abf9a224f0ef61b45/resource.tar.gz#test.test_pg-select_win_first_last_value-default.txt-Debug_/opt.yql_patched"
"checksum": "effe35a51776e4d89dea0a81611a11c9",
"size": 1380,
"uri": "https://{canondata_backend}/1889210/86d0f0a9f5fd231dca8140f5809c568e15366735/resource.tar.gz#test.test_pg-select_win_first_last_value-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg-select_win_first_last_value-default.txt-Plan]": [
Expand Down
12 changes: 6 additions & 6 deletions ydb/library/yql/tests/sql/dq_file/part13/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1831,9 +1831,9 @@
],
"test.test[pg-select_win_expr_order-default.txt-Debug]": [
{
"checksum": "3fc3d13886e9e2b996d3fb603b380cde",
"size": 2004,
"uri": "https://{canondata_backend}/1936997/93899b3de50fae3f9677baacc98094a7a629590a/resource.tar.gz#test.test_pg-select_win_expr_order-default.txt-Debug_/opt.yql_patched"
"checksum": "55224d81e45d2fc2eca42a9b8cf95b93",
"size": 2010,
"uri": "https://{canondata_backend}/1937027/5c72195b4220737ff2c85cc87e9305d1d3749b17/resource.tar.gz#test.test_pg-select_win_expr_order-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg-select_win_expr_order-default.txt-Plan]": [
Expand All @@ -1853,9 +1853,9 @@
],
"test.test[pg-select_win_max_null-default.txt-Debug]": [
{
"checksum": "d04b83176d3a040b46ae648b74f91da9",
"size": 1629,
"uri": "https://{canondata_backend}/1942525/32a558f2f82aa94ed548a77eaea0657193500581/resource.tar.gz#test.test_pg-select_win_max_null-default.txt-Debug_/opt.yql_patched"
"checksum": "dc123729afee48f41507652765608fa0",
"size": 1635,
"uri": "https://{canondata_backend}/1937027/5c72195b4220737ff2c85cc87e9305d1d3749b17/resource.tar.gz#test.test_pg-select_win_max_null-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg-select_win_max_null-default.txt-Plan]": [
Expand Down
22 changes: 22 additions & 0 deletions ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2003,6 +2003,28 @@
}
],
"test.test[pg-select_subquery_scalar2_qstar-default.txt-Results]": [],
"test.test[pg-sort_nulls_priority_window-default.txt-Analyze]": [
{
"checksum": "a3b64a2cf9903b3868a2dd88a18fc46e",
"size": 922,
"uri": "https://{canondata_backend}/1942525/e7939b4cfb5e85a7bd57688517d44a82bd824253/resource.tar.gz#test.test_pg-sort_nulls_priority_window-default.txt-Analyze_/plan.txt"
}
],
"test.test[pg-sort_nulls_priority_window-default.txt-Debug]": [
{
"checksum": "93d1d2e5578d3026fd2e34be5361ff1b",
"size": 1988,
"uri": "https://{canondata_backend}/1942525/e7939b4cfb5e85a7bd57688517d44a82bd824253/resource.tar.gz#test.test_pg-sort_nulls_priority_window-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg-sort_nulls_priority_window-default.txt-Plan]": [
{
"checksum": "a3b64a2cf9903b3868a2dd88a18fc46e",
"size": 922,
"uri": "https://{canondata_backend}/1942525/e7939b4cfb5e85a7bd57688517d44a82bd824253/resource.tar.gz#test.test_pg-sort_nulls_priority_window-default.txt-Plan_/plan.txt"
}
],
"test.test[pg-sort_nulls_priority_window-default.txt-Results]": [],
"test.test[pg-sublink_having_any-default.txt-Analyze]": [
{
"checksum": "b2a2eb5d6b0a138ee924c128fc7738ef",
Expand Down
Loading
Loading