From cce26173b340aa1a00d66319042ed361272095d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 15 Apr 2024 11:51:13 +0200 Subject: [PATCH 01/12] feat(ir): make impure ibis.random() and ibis.uuid() functions return unique node instances --- ibis/backends/bigquery/compiler.py | 5 ++-- ibis/backends/clickhouse/compiler.py | 8 ++++-- ibis/backends/datafusion/compiler.py | 1 - ibis/backends/duckdb/compiler.py | 7 +++++- ibis/backends/flink/compiler.py | 2 -- ibis/backends/impala/compiler.py | 1 - ibis/backends/mssql/compiler.py | 5 ++-- ibis/backends/pandas/executor.py | 8 ------ ibis/backends/postgres/compiler.py | 4 ++- ibis/backends/snowflake/compiler.py | 6 +++-- ibis/backends/sql/compiler.py | 9 ++++++- ibis/backends/sql/rewrites.py | 8 +++++- ibis/backends/sqlite/compiler.py | 3 +-- .../clickhouse-value0/out.sql | 12 +++++++++ .../clickhouse-value1/out.sql | 12 +++++++++ .../datafusion-value0/out.sql | 12 +++++++++ .../datafusion-value1/out.sql | 12 +++++++++ .../duckdb-value0/out.sql | 12 +++++++++ .../duckdb-value1/out.sql | 12 +++++++++ .../flink-value0/out.sql | 12 +++++++++ .../flink-value1/out.sql | 12 +++++++++ .../mssql-value0/out.sql | 12 +++++++++ .../mssql-value1/out.sql | 12 +++++++++ .../mysql-value0/out.sql | 12 +++++++++ .../mysql-value1/out.sql | 12 +++++++++ .../postgres-value0/out.sql | 12 +++++++++ .../postgres-value1/out.sql | 12 +++++++++ .../pyspark-value0/out.sql | 12 +++++++++ .../pyspark-value1/out.sql | 12 +++++++++ .../sqlite-value0/out.sql | 16 ++++++++++++ .../sqlite-value1/out.sql | 12 +++++++++ .../trino-value0/out.sql | 12 +++++++++ .../trino-value1/out.sql | 12 +++++++++ ibis/backends/tests/test_sql.py | 15 +++++++++++ ibis/backends/tests/test_uuid.py | 17 +++++-------- ibis/backends/trino/compiler.py | 1 - ibis/expr/operations/generic.py | 17 +++++++++++-- ibis/expr/operations/tests/test_generic.py | 10 ++++++++ ibis/expr/tests/test_newrels.py | 25 +++++++++++++++++++ ibis/tests/expr/test_table.py | 6 ++++- ibis/tests/expr/test_window_frames.py | 5 ++-- 41 files changed, 364 insertions(+), 43 deletions(-) create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value0/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value0/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value0/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value0/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value0/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value0/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value0/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value0/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value0/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value0/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value1/out.sql diff --git a/ibis/backends/bigquery/compiler.py b/ibis/backends/bigquery/compiler.py index 24d7f2065b35..31aae0746645 100644 --- a/ibis/backends/bigquery/compiler.py +++ b/ibis/backends/bigquery/compiler.py @@ -120,8 +120,6 @@ class BigQueryCompiler(SQLGlotCompiler): ops.RPad: "rpad", ops.Levenshtein: "edit_distance", ops.Modulus: "mod", - ops.RandomScalar: "rand", - ops.RandomUUID: "generate_uuid", ops.RegexReplace: "regexp_replace", ops.RegexSearch: "regexp_contains", ops.Time: "time", @@ -698,3 +696,6 @@ def visit_CountDistinct(self, op, *, arg, where): if where is not None: arg = self.if_(where, arg, NULL) return self.f.count(sge.Distinct(expressions=[arg])) + + def visit_RandomUUID(self, op, **kwargs): + return self.f.generate_uuid() diff --git a/ibis/backends/clickhouse/compiler.py b/ibis/backends/clickhouse/compiler.py index 5b079776328f..6d45dc9ef4d5 100644 --- a/ibis/backends/clickhouse/compiler.py +++ b/ibis/backends/clickhouse/compiler.py @@ -98,8 +98,6 @@ class ClickHouseCompiler(SQLGlotCompiler): ops.NotNull: "isNotNull", ops.NullIf: "nullIf", ops.RStrip: "trimRight", - ops.RandomScalar: "randCanonical", - ops.RandomUUID: "generateUUIDv4", ops.RegexReplace: "replaceRegexpAll", ops.RowNumber: "row_number", ops.StartsWith: "startsWith", @@ -637,6 +635,12 @@ def visit_TimestampRange(self, op, *, start, stop, step): def visit_RegexSplit(self, op, *, arg, pattern): return self.f.splitByRegexp(pattern, self.cast(arg, dt.String(nullable=False))) + def visit_RandomScalar(self, op, **kwargs): + return self.f.randCanonical() + + def visit_RandomUUID(self, op, **kwargs): + return self.f.generateUUIDv4() + @staticmethod def _generate_groups(groups): return groups diff --git a/ibis/backends/datafusion/compiler.py b/ibis/backends/datafusion/compiler.py index 7a9a6c4aea93..35ce655f30b7 100644 --- a/ibis/backends/datafusion/compiler.py +++ b/ibis/backends/datafusion/compiler.py @@ -75,7 +75,6 @@ class DataFusionCompiler(SQLGlotCompiler): ops.Last: "last_value", ops.Median: "median", ops.StringLength: "character_length", - ops.RandomUUID: "uuid", ops.RegexSplit: "regex_split", ops.EndsWith: "ends_with", ops.ArrayIntersect: "array_intersect", diff --git a/ibis/backends/duckdb/compiler.py b/ibis/backends/duckdb/compiler.py index c7d4cd3c85eb..6b67e19983a1 100644 --- a/ibis/backends/duckdb/compiler.py +++ b/ibis/backends/duckdb/compiler.py @@ -48,7 +48,6 @@ class DuckDBCompiler(SQLGlotCompiler): ops.MapMerge: "map_concat", ops.MapValues: "map_values", ops.Mode: "mode", - ops.RandomUUID: "uuid", ops.TimeFromHMS: "make_time", ops.TypeOf: "typeof", ops.GeoPoint: "st_point", @@ -418,3 +417,9 @@ def visit_StructField(self, op, *, arg, field): expression=sg.to_identifier(field, quoted=self.quoted), ) return super().visit_StructField(op, arg=arg, field=field) + + def visit_RandomScalar(self, op, **kwargs): + return self.f.random() + + def visit_RandomUUID(self, op, **kwargs): + return self.f.uuid() diff --git a/ibis/backends/flink/compiler.py b/ibis/backends/flink/compiler.py index fefff21e39e1..2ba669c761f4 100644 --- a/ibis/backends/flink/compiler.py +++ b/ibis/backends/flink/compiler.py @@ -79,8 +79,6 @@ class FlinkCompiler(SQLGlotCompiler): ops.MapKeys: "map_keys", ops.MapValues: "map_values", ops.Power: "power", - ops.RandomScalar: "rand", - ops.RandomUUID: "uuid", ops.RegexSearch: "regexp", ops.StrRight: "right", ops.StringLength: "char_length", diff --git a/ibis/backends/impala/compiler.py b/ibis/backends/impala/compiler.py index 9561319d0d2a..193370decaf9 100644 --- a/ibis/backends/impala/compiler.py +++ b/ibis/backends/impala/compiler.py @@ -76,7 +76,6 @@ class ImpalaCompiler(SQLGlotCompiler): ops.Hash: "fnv_hash", ops.LStrip: "ltrim", ops.Ln: "ln", - ops.RandomUUID: "uuid", ops.RStrip: "rtrim", ops.Strip: "trim", ops.TypeOf: "typeof", diff --git a/ibis/backends/mssql/compiler.py b/ibis/backends/mssql/compiler.py index c2c89124a31d..af7f7fbe6b32 100644 --- a/ibis/backends/mssql/compiler.py +++ b/ibis/backends/mssql/compiler.py @@ -129,8 +129,6 @@ class MSSQLCompiler(SQLGlotCompiler): ops.Ln: "log", ops.Log10: "log10", ops.Power: "power", - ops.RandomScalar: "rand", - ops.RandomUUID: "newid", ops.Repeat: "replicate", ops.Reverse: "reverse", ops.StringAscii: "ascii", @@ -172,6 +170,9 @@ def _minimize_spec(start, end, spec): return None return spec + def visit_RandomUUID(self, op, **kwargs): + return self.f.newid() + def visit_StringLength(self, op, *, arg): """The MSSQL LEN function doesn't count trailing spaces. diff --git a/ibis/backends/pandas/executor.py b/ibis/backends/pandas/executor.py index a97aa8512813..9d920010d63d 100644 --- a/ibis/backends/pandas/executor.py +++ b/ibis/backends/pandas/executor.py @@ -80,14 +80,6 @@ def visit(cls, op: ops.Cast, arg, to): else: return PandasConverter.convert_scalar(arg, to) - @classmethod - def visit(cls, op: ops.TypeOf, arg): - raise OperationNotDefinedError("TypeOf is not implemented") - - @classmethod - def visit(cls, op: ops.RandomScalar): - raise OperationNotDefinedError("RandomScalar is not implemented") - @classmethod def visit(cls, op: ops.Greatest, arg): return cls.columnwise(lambda df: df.max(axis=1), arg) diff --git a/ibis/backends/postgres/compiler.py b/ibis/backends/postgres/compiler.py index 659b5df271e5..665c8ba61779 100644 --- a/ibis/backends/postgres/compiler.py +++ b/ibis/backends/postgres/compiler.py @@ -100,7 +100,6 @@ class PostgresCompiler(SQLGlotCompiler): ops.MapContains: "exist", ops.MapKeys: "akeys", ops.MapValues: "avals", - ops.RandomUUID: "gen_random_uuid", ops.RegexSearch: "regexp_like", ops.TimeFromHMS: "make_time", } @@ -111,6 +110,9 @@ def _aggregate(self, funcname: str, *args, where): return sge.Filter(this=expr, expression=sge.Where(this=where)) return expr + def visit_RandomUUID(self, op, **kwargs): + return self.f.gen_random_uuid() + def visit_Mode(self, op, *, arg, where): expr = self.f.mode() expr = sge.WithinGroup( diff --git a/ibis/backends/snowflake/compiler.py b/ibis/backends/snowflake/compiler.py index 91537d0fd9a5..4f5178f0362f 100644 --- a/ibis/backends/snowflake/compiler.py +++ b/ibis/backends/snowflake/compiler.py @@ -78,7 +78,6 @@ class SnowflakeCompiler(SQLGlotCompiler): ops.Hash: "hash", ops.Median: "median", ops.Mode: "mode", - ops.RandomUUID: "uuid_string", ops.StringToTimestamp: "to_timestamp_tz", ops.TimeFromHMS: "time_from_parts", ops.TimestampFromYMDHMS: "timestamp_from_parts", @@ -241,11 +240,14 @@ def visit_MapLength(self, op, *, arg): def visit_Log(self, op, *, arg, base): return self.f.log(base, arg, dialect=self.dialect) - def visit_RandomScalar(self, op): + def visit_RandomScalar(self, op, **kwargs): return self.f.uniform( self.f.to_double(0.0), self.f.to_double(1.0), self.f.random() ) + def visit_RandomUUID(self, op, **kwargs): + return self.f.uuid_string() + def visit_ApproxMedian(self, op, *, arg, where): return self.agg.approx_percentile(arg, 0.5, where=where) diff --git a/ibis/backends/sql/compiler.py b/ibis/backends/sql/compiler.py index b231c1c3e76f..7adbb935af36 100644 --- a/ibis/backends/sql/compiler.py +++ b/ibis/backends/sql/compiler.py @@ -262,7 +262,6 @@ class SQLGlotCompiler(abc.ABC): ops.Power: "pow", ops.RPad: "rpad", ops.Radians: "radians", - ops.RandomScalar: "random", ops.RegexSearch: "regexp_like", ops.RegexSplit: "regexp_split", ops.Repeat: "repeat", @@ -687,6 +686,14 @@ def visit_Round(self, op, *, arg, digits): return sge.Round(this=arg, decimals=digits) return sge.Round(this=arg) + ### Random Noise + + def visit_RandomScalar(self, op, **kwargs): + return self.f.rand() + + def visit_RandomUUID(self, op, **kwargs): + return self.f.uuid() + ### Dtype Dysmorphia def visit_TryCast(self, op, *, arg, to): diff --git a/ibis/backends/sql/rewrites.py b/ibis/backends/sql/rewrites.py index f8efaa19b264..dffe72258f44 100644 --- a/ibis/backends/sql/rewrites.py +++ b/ibis/backends/sql/rewrites.py @@ -152,7 +152,13 @@ def merge_select_select(_, **kwargs): from the inner Select are inlined into the outer Select. """ # don't merge if either the outer or the inner select has window functions - blocking = (ops.WindowFunction, ops.ExistsSubquery, ops.InSubquery, ops.Unnest) + blocking = ( + ops.WindowFunction, + ops.ExistsSubquery, + ops.InSubquery, + ops.Unnest, + ops.Impure, + ) if _.find_below(blocking, filter=ops.Value): return _ if _.parent.find_below(blocking, filter=ops.Value): diff --git a/ibis/backends/sqlite/compiler.py b/ibis/backends/sqlite/compiler.py index 32e67df84951..77677f6b6b01 100644 --- a/ibis/backends/sqlite/compiler.py +++ b/ibis/backends/sqlite/compiler.py @@ -103,7 +103,6 @@ class SQLiteCompiler(SQLGlotCompiler): ops.Mode: "_ibis_mode", ops.Time: "time", ops.Date: "date", - ops.RandomUUID: "uuid", } def _aggregate(self, funcname: str, *args, where): @@ -213,7 +212,7 @@ def visit_Clip(self, op, *, arg, lower, upper): return arg - def visit_RandomScalar(self, op): + def visit_RandomScalar(self, op, **kwargs): return 0.5 + self.f.random() / sge.Literal.number(float(-1 << 64)) def visit_Cot(self, op, *, arg): diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value0/out.sql new file mode 100644 index 000000000000..d8f3ac64531b --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + randCanonical() AS "y", + randCanonical() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value1/out.sql new file mode 100644 index 000000000000..f2ed2707b0f5 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + generateUUIDv4() AS "y", + generateUUIDv4() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value0/out.sql new file mode 100644 index 000000000000..e35fcb21ec50 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + RANDOM() AS "y", + RANDOM() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value1/out.sql new file mode 100644 index 000000000000..8d377a29229e --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + UUID() AS "y", + UUID() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value0/out.sql new file mode 100644 index 000000000000..e35fcb21ec50 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + RANDOM() AS "y", + RANDOM() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value1/out.sql new file mode 100644 index 000000000000..8d377a29229e --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + UUID() AS "y", + UUID() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value0/out.sql new file mode 100644 index 000000000000..551447b11ff7 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + `t1`.`x`, + `t1`.`y`, + `t1`.`z`, + IF(`t1`.`y` = `t1`.`z`, 'big', 'small') AS `size` +FROM ( + SELECT + `t0`.`x`, + RAND() AS `y`, + RAND() AS `z` + FROM `t` AS `t0` +) AS `t1` \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value1/out.sql new file mode 100644 index 000000000000..fb13922470e3 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + `t1`.`x`, + `t1`.`y`, + `t1`.`z`, + IF(`t1`.`y` = `t1`.`z`, 'big', 'small') AS `size` +FROM ( + SELECT + `t0`.`x`, + UUID() AS `y`, + UUID() AS `z` + FROM `t` AS `t0` +) AS `t1` \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value0/out.sql new file mode 100644 index 000000000000..6944df66595d --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + [t1].[x], + [t1].[y], + [t1].[z], + IIF([t1].[y] = [t1].[z], 'big', 'small') AS [size] +FROM ( + SELECT + [t0].[x], + RAND() AS [y], + RAND() AS [z] + FROM [t] AS [t0] +) AS [t1] \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value1/out.sql new file mode 100644 index 000000000000..3c9c8aa49168 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + [t1].[x], + [t1].[y], + [t1].[z], + IIF([t1].[y] = [t1].[z], 'big', 'small') AS [size] +FROM ( + SELECT + [t0].[x], + NEWID() AS [y], + NEWID() AS [z] + FROM [t] AS [t0] +) AS [t1] \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value0/out.sql new file mode 100644 index 000000000000..f7be3c965695 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + `t1`.`x`, + `t1`.`y`, + `t1`.`z`, + CASE WHEN `t1`.`y` = `t1`.`z` THEN 'big' ELSE 'small' END AS `size` +FROM ( + SELECT + `t0`.`x`, + RAND() AS `y`, + RAND() AS `z` + FROM `t` AS `t0` +) AS `t1` \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value1/out.sql new file mode 100644 index 000000000000..72b30c387407 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + `t1`.`x`, + `t1`.`y`, + `t1`.`z`, + CASE WHEN `t1`.`y` = `t1`.`z` THEN 'big' ELSE 'small' END AS `size` +FROM ( + SELECT + `t0`.`x`, + UUID() AS `y`, + UUID() AS `z` + FROM `t` AS `t0` +) AS `t1` \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value0/out.sql new file mode 100644 index 000000000000..e35fcb21ec50 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + RANDOM() AS "y", + RANDOM() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value1/out.sql new file mode 100644 index 000000000000..2da38dc85327 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + GEN_RANDOM_UUID() AS "y", + GEN_RANDOM_UUID() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value0/out.sql new file mode 100644 index 000000000000..551447b11ff7 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + `t1`.`x`, + `t1`.`y`, + `t1`.`z`, + IF(`t1`.`y` = `t1`.`z`, 'big', 'small') AS `size` +FROM ( + SELECT + `t0`.`x`, + RAND() AS `y`, + RAND() AS `z` + FROM `t` AS `t0` +) AS `t1` \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value1/out.sql new file mode 100644 index 000000000000..fb13922470e3 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + `t1`.`x`, + `t1`.`y`, + `t1`.`z`, + IF(`t1`.`y` = `t1`.`z`, 'big', 'small') AS `size` +FROM ( + SELECT + `t0`.`x`, + UUID() AS `y`, + UUID() AS `z` + FROM `t` AS `t0` +) AS `t1` \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value0/out.sql new file mode 100644 index 000000000000..413e3e0c9bfa --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value0/out.sql @@ -0,0 +1,16 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + IIF("t1"."y" = "t1"."z", 'big', 'small') AS "size" +FROM ( + SELECT + "t0"."x", + 0.5 + ( + CAST(RANDOM() AS REAL) / -1.8446744073709552e+19 + ) AS "y", + 0.5 + ( + CAST(RANDOM() AS REAL) / -1.8446744073709552e+19 + ) AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value1/out.sql new file mode 100644 index 000000000000..8bb07bf993b5 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + IIF("t1"."y" = "t1"."z", 'big', 'small') AS "size" +FROM ( + SELECT + "t0"."x", + UUID() AS "y", + UUID() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value0/out.sql new file mode 100644 index 000000000000..2f2064f9323b --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + IF("t1"."y" = "t1"."z", 'big', 'small') AS "size" +FROM ( + SELECT + "t0"."x", + RAND() AS "y", + RAND() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value1/out.sql new file mode 100644 index 000000000000..6884f6004e46 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + IF("t1"."y" = "t1"."z", 'big', 'small') AS "size" +FROM ( + SELECT + "t0"."x", + UUID() AS "y", + UUID() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/test_sql.py b/ibis/backends/tests/test_sql.py index 0bf6ce5304ac..ebde1b1761a4 100644 --- a/ibis/backends/tests/test_sql.py +++ b/ibis/backends/tests/test_sql.py @@ -177,3 +177,18 @@ def test_union_generates_predictable_aliases(con): expr = ibis.union(sub1, sub2) df = con.execute(expr) assert len(df) == 2 + + +@pytest.mark.never( + ["pandas", "dask", "polars"], + reason="not SQL", + raises=NotImplementedError, +) +@pytest.mark.parametrize("value", [ibis.random(), ibis.uuid()]) +def test_selects_with_impure_operations_not_merged(con, snapshot, value): + t = ibis.table({"x": "int64", "y": "float64"}, name="t") + t = t.mutate(y=value, z=value) + t = t.mutate(size=(t.y == t.z).ifelse("big", "small")) + + sql = str(ibis.to_sql(t, dialect=con.name)) + snapshot.assert_match(sql, "out.sql") diff --git a/ibis/backends/tests/test_uuid.py b/ibis/backends/tests/test_uuid.py index 6caf06ab1dde..8a751829ccc1 100644 --- a/ibis/backends/tests/test_uuid.py +++ b/ibis/backends/tests/test_uuid.py @@ -42,18 +42,12 @@ def test_uuid_literal(con, backend): @pytest.mark.notimpl( - [ - "druid", - "exasol", - "mysql", - "oracle", - "polars", - "pyspark", - "risingwave", - ], + ["druid", "exasol", "oracle", "polars", "pyspark", "risingwave", "pandas", "dask"], raises=com.OperationNotDefinedError, ) -@pytest.mark.notimpl(["pandas", "dask"], raises=ValueError) +@pytest.mark.broken( + ["mysql"], raises=AssertionError, reason="MySQL generates version 1 UUIDs" +) def test_uuid_function(con): obj = con.execute(ibis.uuid()) assert isinstance(obj, uuid.UUID) @@ -69,10 +63,11 @@ def test_uuid_function(con): "polars", "pyspark", "risingwave", + "pandas", + "dask", ], raises=com.OperationNotDefinedError, ) -@pytest.mark.notimpl(["pandas", "dask"], raises=ValueError) def test_uuid_unique_each_row(con): expr = ( con.tables.functional_alltypes.mutate(uuid=ibis.uuid()).limit(2).uuid.nunique() diff --git a/ibis/backends/trino/compiler.py b/ibis/backends/trino/compiler.py index bcbb0046457f..75efe46e7a8c 100644 --- a/ibis/backends/trino/compiler.py +++ b/ibis/backends/trino/compiler.py @@ -80,7 +80,6 @@ class TrinoCompiler(SQLGlotCompiler): ops.ExtractPath: "url_extract_path", ops.ExtractFragment: "url_extract_fragment", ops.ArrayPosition: "array_position", - ops.RandomUUID: "uuid", } def _aggregate(self, funcname: str, *args, where): diff --git a/ibis/expr/operations/generic.py b/ibis/expr/operations/generic.py index 1f83791c4595..754029e292b5 100644 --- a/ibis/expr/operations/generic.py +++ b/ibis/expr/operations/generic.py @@ -183,6 +183,17 @@ class Constant(Scalar, Singleton): shape = ds.scalar +@public +class Impure(Value): + _counter = itertools.count() + uid: Optional[int] = None + + def __init__(self, uid, **kwargs): + if uid is None: + uid = next(self._counter) + super().__init__(uid=uid, **kwargs) + + @public class TimestampNow(Constant): dtype = dt.timestamp @@ -194,13 +205,15 @@ class DateNow(Constant): @public -class RandomScalar(Constant): +class RandomScalar(Impure): dtype = dt.float64 + shape = ds.scalar @public -class RandomUUID(Constant): +class RandomUUID(Impure): dtype = dt.uuid + shape = ds.scalar @public diff --git a/ibis/expr/operations/tests/test_generic.py b/ibis/expr/operations/tests/test_generic.py index 8dd7ff12ebcf..c7f4f17873db 100644 --- a/ibis/expr/operations/tests/test_generic.py +++ b/ibis/expr/operations/tests/test_generic.py @@ -143,3 +143,13 @@ def test_NULL(): assert ops.NULL.dtype is dt.null assert ops.NULL == ops.Literal(None, dt.null) assert ops.NULL is not ops.Literal(None, dt.int8) + + +@pytest.mark.parametrize("op", [ops.RandomScalar, ops.RandomUUID]) +def test_unique_impure_values(op): + assert op() != op() + assert hash(op()) != hash(op()) + + node = op() + other = node.copy() + assert node == other diff --git a/ibis/expr/tests/test_newrels.py b/ibis/expr/tests/test_newrels.py index 6da1ed32d5c6..9635459d4314 100644 --- a/ibis/expr/tests/test_newrels.py +++ b/ibis/expr/tests/test_newrels.py @@ -1606,3 +1606,28 @@ def test_subsequent_order_by_calls(): first = ops.Sort(t, [t.int_col.desc()]).to_expr() second = ops.Sort(first, [first.int_col.asc()]).to_expr() assert ts.equals(second) + + +@pytest.mark.parametrize("func", [ibis.random, ibis.uuid]) +def test_impure_operation_dereferencing(func): + t = ibis.table({"x": "int64"}, name="t") + + impure = func() + t1 = t.mutate(y=impure) + t2 = t1.mutate(z=impure.cast("string")) + + expected = ops.Project( + parent=t1, + values={"x": t1.x, "y": t1.y, "z": t1.y.cast("string")}, + ) + assert t2.op() == expected + + v1 = func() + v2 = func() + + t1 = t.mutate(y=v1) + t2 = t1.mutate(z=v2.cast("string")) + expected = ops.Project( + parent=t1, values={"x": t1.x, "y": t1.y, "z": v2.cast("string")} + ) + assert t2.op() == expected diff --git a/ibis/tests/expr/test_table.py b/ibis/tests/expr/test_table.py index 627c227cab66..b7f81360d1c9 100644 --- a/ibis/tests/expr/test_table.py +++ b/ibis/tests/expr/test_table.py @@ -562,11 +562,15 @@ def test_order_by_asc_deferred_sort_key(table): assert_equal(result, expected2) +# different instantiations create unique objects +rand = ibis.random() + + @pytest.mark.parametrize( ("key", "expected"), [ param(ibis.NA, ibis.NA.op(), id="na"), - param(ibis.random(), ibis.random().op(), id="random"), + param(rand, rand.op(), id="random"), param(1.0, ibis.literal(1.0).op(), id="float"), param(ibis.literal("a"), ibis.literal("a").op(), id="string"), param(ibis.literal([1, 2, 3]), ibis.literal([1, 2, 3]).op(), id="array"), diff --git a/ibis/tests/expr/test_window_frames.py b/ibis/tests/expr/test_window_frames.py index f44b739e1cc8..2e88f2c2cac0 100644 --- a/ibis/tests/expr/test_window_frames.py +++ b/ibis/tests/expr/test_window_frames.py @@ -246,7 +246,8 @@ def test_window_api_supports_scalar_order_by(t): ) assert expr == expected - window = ibis.window(order_by=ibis.random()) + rand = ibis.random() + window = ibis.window(order_by=rand) expr = t.a.sum().over(window).op() expected = ops.WindowFunction( t.a.sum(), @@ -254,7 +255,7 @@ def test_window_api_supports_scalar_order_by(t): start=None, end=None, group_by=(), - order_by=[ibis.random()], + order_by=[rand], ) assert expr == expected From 2355aaf7713024a0edbcbae1e75d854b3457e48c Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 15 Apr 2024 12:56:34 -0400 Subject: [PATCH 02/12] chore(sql): update snapshots --- .../druid-value0/out.sql | 12 ++++++++++++ .../druid-value1/out.sql | 12 ++++++++++++ .../impala-value1/out.sql | 12 ++++++++++++ .../oracle-value1/out.sql | 12 ++++++++++++ .../risingwave-value0/out.sql | 12 ++++++++++++ 5 files changed, 60 insertions(+) create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value0/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value1/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/risingwave-value0/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value0/out.sql new file mode 100644 index 000000000000..e35fcb21ec50 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + RANDOM() AS "y", + RANDOM() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value1/out.sql new file mode 100644 index 000000000000..8d377a29229e --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + UUID() AS "y", + UUID() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value1/out.sql new file mode 100644 index 000000000000..fb13922470e3 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + `t1`.`x`, + `t1`.`y`, + `t1`.`z`, + IF(`t1`.`y` = `t1`.`z`, 'big', 'small') AS `size` +FROM ( + SELECT + `t0`.`x`, + UUID() AS `y`, + UUID() AS `z` + FROM `t` AS `t0` +) AS `t1` \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value1/out.sql new file mode 100644 index 000000000000..f21630bdfb49 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value1/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + UUID() AS "y", + UUID() AS "z" + FROM "t" "t0" +) "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/risingwave-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/risingwave-value0/out.sql new file mode 100644 index 000000000000..e35fcb21ec50 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/risingwave-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + RANDOM() AS "y", + RANDOM() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file From 9d2df0abe44624d167a03ff2d60d8d5d5f87d2b1 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:18:37 -0400 Subject: [PATCH 03/12] fixup! chore(sql): update snapshots --- .../oracle-value0/out.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value0/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value0/out.sql new file mode 100644 index 000000000000..03f3cec9772d --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" +FROM ( + SELECT + "t0"."x", + DBMS_RANDOM.VALUE() AS "y", + DBMS_RANDOM.VALUE() AS "z" + FROM "t" "t0" +) "t1" \ No newline at end of file From 84372c27b8c3eeee95bebcec74247edfabad1aca Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:18:47 -0400 Subject: [PATCH 04/12] chore(oracle): fix impls --- ibis/backends/oracle/compiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibis/backends/oracle/compiler.py b/ibis/backends/oracle/compiler.py index 4645467ace92..9fd0a76890a7 100644 --- a/ibis/backends/oracle/compiler.py +++ b/ibis/backends/oracle/compiler.py @@ -80,6 +80,7 @@ class OracleCompiler(SQLGlotCompiler): ops.ExtractWeekOfYear, ops.ExtractDayOfYear, ops.RowID, + ops.RandomUUID, ) ) @@ -221,7 +222,7 @@ def visit_Log(self, op, *, arg, base): def visit_IsInf(self, op, *, arg): return arg.isin(self.POS_INF, self.NEG_INF) - def visit_RandomScalar(self, op): + def visit_RandomScalar(self, op, **_): # Not using FuncGen here because of dotted function call return sg.func("dbms_random.value") From 6f9047ae38bb21413d77aa6200da38900d53975f Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:20:39 -0400 Subject: [PATCH 05/12] test(mysql): enable xpassing uuid test --- ibis/backends/tests/test_uuid.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/ibis/backends/tests/test_uuid.py b/ibis/backends/tests/test_uuid.py index 8a751829ccc1..727f1e0db438 100644 --- a/ibis/backends/tests/test_uuid.py +++ b/ibis/backends/tests/test_uuid.py @@ -55,17 +55,7 @@ def test_uuid_function(con): @pytest.mark.notimpl( - [ - "druid", - "exasol", - "mysql", - "oracle", - "polars", - "pyspark", - "risingwave", - "pandas", - "dask", - ], + ["druid", "exasol", "oracle", "polars", "pyspark", "risingwave", "pandas", "dask"], raises=com.OperationNotDefinedError, ) def test_uuid_unique_each_row(con): From 89f37bbce2598159eca08a37a1aa52fafc9daf1c Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:24:02 -0400 Subject: [PATCH 06/12] test(risingwave): xfail for missing uuid impl --- ibis/backends/tests/test_sql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ibis/backends/tests/test_sql.py b/ibis/backends/tests/test_sql.py index ebde1b1761a4..52794d6c74f7 100644 --- a/ibis/backends/tests/test_sql.py +++ b/ibis/backends/tests/test_sql.py @@ -184,6 +184,7 @@ def test_union_generates_predictable_aliases(con): reason="not SQL", raises=NotImplementedError, ) +@pytest.mark.notimpl(["risingwave"], raises=exc.OperationNotDefinedError) @pytest.mark.parametrize("value", [ibis.random(), ibis.uuid()]) def test_selects_with_impure_operations_not_merged(con, snapshot, value): t = ibis.table({"x": "int64", "y": "float64"}, name="t") From da410dee7d39366981f8bd26c69b9d467a36a104 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:26:05 -0400 Subject: [PATCH 07/12] test(impala): passing missing random arguments --- ibis/backends/impala/compiler.py | 2 +- .../impala-value0/out.sql | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value0/out.sql diff --git a/ibis/backends/impala/compiler.py b/ibis/backends/impala/compiler.py index 193370decaf9..fa2b0b224108 100644 --- a/ibis/backends/impala/compiler.py +++ b/ibis/backends/impala/compiler.py @@ -145,7 +145,7 @@ def visit_CountDistinct(self, op, *, arg, where): def visit_Xor(self, op, *, left, right): return sg.and_(sg.or_(left, right), sg.not_(sg.and_(left, right))) - def visit_RandomScalar(self, op): + def visit_RandomScalar(self, op, **_): return self.f.rand(self.f.utc_to_unix_micros(self.f.utc_timestamp())) def visit_DayOfWeekIndex(self, op, *, arg): diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value0/out.sql new file mode 100644 index 000000000000..7d037e909fcd --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value0/out.sql @@ -0,0 +1,12 @@ +SELECT + `t1`.`x`, + `t1`.`y`, + `t1`.`z`, + IF(`t1`.`y` = `t1`.`z`, 'big', 'small') AS `size` +FROM ( + SELECT + `t0`.`x`, + RAND(UTC_TO_UNIX_MICROS(UTC_TIMESTAMP())) AS `y`, + RAND(UTC_TO_UNIX_MICROS(UTC_TIMESTAMP())) AS `z` + FROM `t` AS `t0` +) AS `t1` \ No newline at end of file From 7d9a2fcd59229d230aba09c49626183901d8b400 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:36:26 -0400 Subject: [PATCH 08/12] chore(snapshots): regen with more meaningful test ids --- ibis/backends/druid/compiler.py | 1 + ibis/backends/exasol/compiler.py | 1 + .../{flink-value0 => bigquery-random}/out.sql | 0 .../{pyspark-value1 => bigquery-uuid}/out.sql | 4 ++-- .../{clickhouse-value0 => clickhouse-random}/out.sql | 0 .../{clickhouse-value1 => clickhouse-uuid}/out.sql | 0 .../{datafusion-value0 => datafusion-random}/out.sql | 0 .../{datafusion-value1 => datafusion-uuid}/out.sql | 0 .../{druid-value0 => druid-random}/out.sql | 0 .../{duckdb-value0 => duckdb-random}/out.sql | 0 .../{druid-value1 => duckdb-uuid}/out.sql | 0 .../{postgres-value0 => exasol-random}/out.sql | 0 .../{impala-value0 => impala-random}/out.sql | 0 .../{flink-value1 => impala-uuid}/out.sql | 0 .../{mssql-value0 => mssql-random}/out.sql | 0 .../{mssql-value1 => mssql-uuid}/out.sql | 0 .../{mysql-value0 => mysql-random}/out.sql | 0 .../{mysql-value1 => mysql-uuid}/out.sql | 0 .../{oracle-value0 => oracle-random}/out.sql | 0 .../oracle-value1/out.sql | 12 ------------ .../{risingwave-value0 => postgres-random}/out.sql | 0 .../{postgres-value1 => postgres-uuid}/out.sql | 0 .../{pyspark-value0 => pyspark-random}/out.sql | 0 .../{impala-value1 => pyspark-uuid}/out.sql | 0 .../{duckdb-value1 => risingwave-random}/out.sql | 4 ++-- .../snowflake-random/out.sql | 12 ++++++++++++ .../snowflake-uuid/out.sql | 12 ++++++++++++ .../{sqlite-value0 => sqlite-random}/out.sql | 0 .../{sqlite-value1 => sqlite-uuid}/out.sql | 0 .../{trino-value0 => trino-random}/out.sql | 0 .../{trino-value1 => trino-uuid}/out.sql | 0 ibis/backends/tests/test_sql.py | 12 +++++++++++- 32 files changed, 41 insertions(+), 17 deletions(-) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{flink-value0 => bigquery-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{pyspark-value1 => bigquery-uuid}/out.sql (73%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{clickhouse-value0 => clickhouse-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{clickhouse-value1 => clickhouse-uuid}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{datafusion-value0 => datafusion-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{datafusion-value1 => datafusion-uuid}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{druid-value0 => druid-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{duckdb-value0 => duckdb-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{druid-value1 => duckdb-uuid}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{postgres-value0 => exasol-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{impala-value0 => impala-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{flink-value1 => impala-uuid}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{mssql-value0 => mssql-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{mssql-value1 => mssql-uuid}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{mysql-value0 => mysql-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{mysql-value1 => mysql-uuid}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{oracle-value0 => oracle-random}/out.sql (100%) delete mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value1/out.sql rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{risingwave-value0 => postgres-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{postgres-value1 => postgres-uuid}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{pyspark-value0 => pyspark-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{impala-value1 => pyspark-uuid}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{duckdb-value1 => risingwave-random}/out.sql (80%) create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/snowflake-random/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/snowflake-uuid/out.sql rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{sqlite-value0 => sqlite-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{sqlite-value1 => sqlite-uuid}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{trino-value0 => trino-random}/out.sql (100%) rename ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/{trino-value1 => trino-uuid}/out.sql (100%) diff --git a/ibis/backends/druid/compiler.py b/ibis/backends/druid/compiler.py index ec1b5ae4cfe7..ce4b3db9a26f 100644 --- a/ibis/backends/druid/compiler.py +++ b/ibis/backends/druid/compiler.py @@ -59,6 +59,7 @@ class DruidCompiler(SQLGlotCompiler): ops.Median, ops.MultiQuantile, ops.Quantile, + ops.RandomUUID, ops.RegexReplace, ops.RegexSplit, ops.RowID, diff --git a/ibis/backends/exasol/compiler.py b/ibis/backends/exasol/compiler.py index 1748baa0094c..1fff079b564e 100644 --- a/ibis/backends/exasol/compiler.py +++ b/ibis/backends/exasol/compiler.py @@ -67,6 +67,7 @@ class ExasolCompiler(SQLGlotCompiler): ops.Median, ops.MultiQuantile, ops.Quantile, + ops.RandomUUID, ops.ReductionVectorizedUDF, ops.RegexExtract, ops.RegexReplace, diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/bigquery-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/bigquery-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/bigquery-uuid/out.sql similarity index 73% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/bigquery-uuid/out.sql index fb13922470e3..2e9d56892510 100644 --- a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value1/out.sql +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/bigquery-uuid/out.sql @@ -6,7 +6,7 @@ SELECT FROM ( SELECT `t0`.`x`, - UUID() AS `y`, - UUID() AS `z` + generate_uuid() AS `y`, + generate_uuid() AS `z` FROM `t` AS `t0` ) AS `t1` \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-uuid/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/clickhouse-uuid/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-uuid/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/datafusion-uuid/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-uuid/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/druid-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-uuid/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/exasol-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/exasol-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-uuid/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-uuid/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-uuid/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-uuid/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-uuid/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mysql-uuid/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value1/out.sql deleted file mode 100644 index f21630bdfb49..000000000000 --- a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/oracle-value1/out.sql +++ /dev/null @@ -1,12 +0,0 @@ -SELECT - "t1"."x", - "t1"."y", - "t1"."z", - CASE WHEN "t1"."y" = "t1"."z" THEN 'big' ELSE 'small' END AS "size" -FROM ( - SELECT - "t0"."x", - UUID() AS "y", - UUID() AS "z" - FROM "t" "t0" -) "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/risingwave-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/risingwave-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-uuid/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/postgres-uuid/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-uuid/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/impala-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/pyspark-uuid/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/risingwave-random/out.sql similarity index 80% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/risingwave-random/out.sql index 8d377a29229e..e35fcb21ec50 100644 --- a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/duckdb-value1/out.sql +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/risingwave-random/out.sql @@ -6,7 +6,7 @@ SELECT FROM ( SELECT "t0"."x", - UUID() AS "y", - UUID() AS "z" + RANDOM() AS "y", + RANDOM() AS "z" FROM "t" AS "t0" ) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/snowflake-random/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/snowflake-random/out.sql new file mode 100644 index 000000000000..96d88a6d5bba --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/snowflake-random/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + IFF("t1"."y" = "t1"."z", 'big', 'small') AS "size" +FROM ( + SELECT + "t0"."x", + UNIFORM(TO_DOUBLE(0.0), TO_DOUBLE(1.0), RANDOM()) AS "y", + UNIFORM(TO_DOUBLE(0.0), TO_DOUBLE(1.0), RANDOM()) AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/snowflake-uuid/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/snowflake-uuid/out.sql new file mode 100644 index 000000000000..58e7190a5e37 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/snowflake-uuid/out.sql @@ -0,0 +1,12 @@ +SELECT + "t1"."x", + "t1"."y", + "t1"."z", + IFF("t1"."y" = "t1"."z", 'big', 'small') AS "size" +FROM ( + SELECT + "t0"."x", + UUID_STRING() AS "y", + UUID_STRING() AS "z" + FROM "t" AS "t0" +) AS "t1" \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-uuid/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/sqlite-uuid/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value0/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-random/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value0/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-random/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value1/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-uuid/out.sql similarity index 100% rename from ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-value1/out.sql rename to ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/trino-uuid/out.sql diff --git a/ibis/backends/tests/test_sql.py b/ibis/backends/tests/test_sql.py index 52794d6c74f7..0e1518d353c1 100644 --- a/ibis/backends/tests/test_sql.py +++ b/ibis/backends/tests/test_sql.py @@ -185,7 +185,17 @@ def test_union_generates_predictable_aliases(con): raises=NotImplementedError, ) @pytest.mark.notimpl(["risingwave"], raises=exc.OperationNotDefinedError) -@pytest.mark.parametrize("value", [ibis.random(), ibis.uuid()]) +@pytest.mark.parametrize( + "value", + [ + param(ibis.random(), id="random"), + param( + ibis.uuid(), + marks=pytest.mark.notimpl(["exasol"], raises=exc.OperationNotDefinedError), + id="uuid", + ), + ], +) def test_selects_with_impure_operations_not_merged(con, snapshot, value): t = ibis.table({"x": "int64", "y": "float64"}, name="t") t = t.mutate(y=value, z=value) From 4e271d923bb51b89fd7cb5943b7db2bae6d4db27 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:41:36 -0400 Subject: [PATCH 09/12] test(merging): xfail things --- ibis/backends/tests/test_sql.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ibis/backends/tests/test_sql.py b/ibis/backends/tests/test_sql.py index 0e1518d353c1..b5322c5be3cc 100644 --- a/ibis/backends/tests/test_sql.py +++ b/ibis/backends/tests/test_sql.py @@ -184,14 +184,16 @@ def test_union_generates_predictable_aliases(con): reason="not SQL", raises=NotImplementedError, ) -@pytest.mark.notimpl(["risingwave"], raises=exc.OperationNotDefinedError) @pytest.mark.parametrize( "value", [ param(ibis.random(), id="random"), param( ibis.uuid(), - marks=pytest.mark.notimpl(["exasol"], raises=exc.OperationNotDefinedError), + marks=pytest.mark.notimpl( + ["exasol", "risingwave", "druid", "oracle"], + raises=exc.OperationNotDefinedError, + ), id="uuid", ), ], From a5783c2509f885ce07989aed7f692cfc53aaf1a7 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 15 Apr 2024 17:02:54 -0400 Subject: [PATCH 10/12] test(pyspark): disable uuid --- ibis/backends/pyspark/compiler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ibis/backends/pyspark/compiler.py b/ibis/backends/pyspark/compiler.py index c9fa54b87bc8..eb8de0c56b17 100644 --- a/ibis/backends/pyspark/compiler.py +++ b/ibis/backends/pyspark/compiler.py @@ -57,6 +57,7 @@ class PySparkCompiler(SQLGlotCompiler): ( ops.RowID, ops.TimestampBucket, + ops.RandomUUID, ) ) From 9b4877df45daf41cff5b70b72077e82884e420a2 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 15 Apr 2024 17:20:22 -0400 Subject: [PATCH 11/12] test(flink): snapshots --- .../flink-random/out.sql | 12 ++++++++++++ .../flink-uuid/out.sql | 12 ++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-random/out.sql create mode 100644 ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-uuid/out.sql diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-random/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-random/out.sql new file mode 100644 index 000000000000..551447b11ff7 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-random/out.sql @@ -0,0 +1,12 @@ +SELECT + `t1`.`x`, + `t1`.`y`, + `t1`.`z`, + IF(`t1`.`y` = `t1`.`z`, 'big', 'small') AS `size` +FROM ( + SELECT + `t0`.`x`, + RAND() AS `y`, + RAND() AS `z` + FROM `t` AS `t0` +) AS `t1` \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-uuid/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-uuid/out.sql new file mode 100644 index 000000000000..fb13922470e3 --- /dev/null +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/flink-uuid/out.sql @@ -0,0 +1,12 @@ +SELECT + `t1`.`x`, + `t1`.`y`, + `t1`.`z`, + IF(`t1`.`y` = `t1`.`z`, 'big', 'small') AS `size` +FROM ( + SELECT + `t0`.`x`, + UUID() AS `y`, + UUID() AS `z` + FROM `t` AS `t0` +) AS `t1` \ No newline at end of file From d690f903e1b26e0e6fdac4d3d0b1533bd2b6a5d0 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 15 Apr 2024 17:31:02 -0400 Subject: [PATCH 12/12] test(pyspark): xfail merge on uuid test --- ibis/backends/tests/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis/backends/tests/test_sql.py b/ibis/backends/tests/test_sql.py index b5322c5be3cc..097bbb9cb457 100644 --- a/ibis/backends/tests/test_sql.py +++ b/ibis/backends/tests/test_sql.py @@ -191,7 +191,7 @@ def test_union_generates_predictable_aliases(con): param( ibis.uuid(), marks=pytest.mark.notimpl( - ["exasol", "risingwave", "druid", "oracle"], + ["exasol", "risingwave", "druid", "oracle", "pyspark"], raises=exc.OperationNotDefinedError, ), id="uuid",