diff --git a/docs/developer-guide/ksqldb-reference/create-table-as-select.md b/docs/developer-guide/ksqldb-reference/create-table-as-select.md index 668b9d51d540..9c4a7e1b8efe 100644 --- a/docs/developer-guide/ksqldb-reference/create-table-as-select.md +++ b/docs/developer-guide/ksqldb-reference/create-table-as-select.md @@ -55,14 +55,15 @@ See [Partition Data to Enable Joins](../joins/partition-data.md) for more inform correctly partition your data for joins. The primary key of the resulting table is determined by the following rules, in order of priority: - 1. if the query has a `GROUP BY`: - 1. if the `GROUP BY` is on a single source column reference, the primary key will match the - name, type and contents of the source column. - 1. if the `GROUP BY` is any other single expression, the primary key will have a system + 1. if the query has a `GROUP BY`, then the resulting number of primary key columns will match the + number of grouping expressions. For each grouping expression: + 1. if the grouping expression is a single source column reference, the corresponding primary key + column will match the name, type and contents of the source column. + 1. if the grouping expression is a reference to a field within a `STRUCT`-type column, then the + corresponding primary key column will match the name, type, and contents of the `STRUCT` field. + 1. if the `GROUP BY` is any other expression, the primary key will have a system generated name, unless you provide an alias in the projection, and will match the type and contents of the result of the expression. - 1. otherwise, the primary key will have a system generated name, and will be of type `STRING` - and contain the grouping expression concatenated together. 1. if the query has a join see [Join Synthetic Key Columns](../joins/synthetic-keys) for more info. 1. otherwise, the primary key will match the name, unless you provide an alias in the projection, and type of the source table's primary key. diff --git a/ksqldb-common/src/main/java/io/confluent/ksql/util/KsqlConfig.java b/ksqldb-common/src/main/java/io/confluent/ksql/util/KsqlConfig.java index aae9ca6615f0..95351a8b7114 100644 --- a/ksqldb-common/src/main/java/io/confluent/ksql/util/KsqlConfig.java +++ b/ksqldb-common/src/main/java/io/confluent/ksql/util/KsqlConfig.java @@ -139,11 +139,6 @@ public class KsqlConfig extends AbstractConfig { + "in interactive mode. Once this limit is reached, any further persistent queries will not " + "be accepted."; - public static final String KSQL_MULTICOL_KEY_FORMAT_ENABLED = "ksql.multicol.key.format.enabled"; - public static final Boolean KSQL_MULTICOL_KEY_FORMAT_ENABLED_DEFAULT = false; - public static final String KSQL_MULTICOL_KEY_FORMAT_ENABLED_DOC = - "Feature flag for multi-column keys"; - public static final String KSQL_DEFAULT_KEY_FORMAT_CONFIG = "ksql.persistence.default.format.key"; private static final String KSQL_DEFAULT_KEY_FORMAT_DEFAULT = "KAFKA"; private static final String KSQL_DEFAULT_KEY_FORMAT_DOC = @@ -612,12 +607,6 @@ private static ConfigDef buildConfigDef(final ConfigGeneration generation) { KSQL_SECURITY_EXTENSION_DEFAULT, ConfigDef.Importance.LOW, KSQL_SECURITY_EXTENSION_DOC - ).define( - KSQL_MULTICOL_KEY_FORMAT_ENABLED, - Type.BOOLEAN, - KSQL_MULTICOL_KEY_FORMAT_ENABLED_DEFAULT, - ConfigDef.Importance.LOW, - KSQL_MULTICOL_KEY_FORMAT_ENABLED_DOC ).define( KSQL_DEFAULT_KEY_FORMAT_CONFIG, Type.STRING, diff --git a/ksqldb-engine/src/main/java/io/confluent/ksql/planner/LogicalPlanner.java b/ksqldb-engine/src/main/java/io/confluent/ksql/planner/LogicalPlanner.java index b1b94a4dff31..0965ddfb67ea 100644 --- a/ksqldb-engine/src/main/java/io/confluent/ksql/planner/LogicalPlanner.java +++ b/ksqldb-engine/src/main/java/io/confluent/ksql/planner/LogicalPlanner.java @@ -76,7 +76,6 @@ import io.confluent.ksql.schema.ksql.LogicalSchema; import io.confluent.ksql.schema.ksql.LogicalSchema.Builder; import io.confluent.ksql.schema.ksql.types.SqlType; -import io.confluent.ksql.schema.ksql.types.SqlTypes; import io.confluent.ksql.serde.FormatFactory; import io.confluent.ksql.serde.FormatInfo; import io.confluent.ksql.serde.KeyFormat; @@ -643,46 +642,16 @@ private LogicalSchema buildAggregateSchema( final Builder builder = LogicalSchema.builder(); - if (ksqlConfig.getBoolean(KsqlConfig.KSQL_MULTICOL_KEY_FORMAT_ENABLED)) { - final ExpressionTypeManager typeManager = - new ExpressionTypeManager(sourceSchema, metaStore); + final ExpressionTypeManager typeManager = + new ExpressionTypeManager(sourceSchema, metaStore); - for (final Expression expression : groupByExps) { - final SqlType keyType = typeManager.getExpressionSqlType(expression); - final ColumnName keyName = selectResolver.apply(expression) - .orElseGet(() -> expression instanceof ColumnReferenceExp - ? ((ColumnReferenceExp) expression).getColumnName() - : ColumnNames.uniqueAliasFor(expression, sourceSchema) - ); - - builder.keyColumn(keyName, keyType); - } - } else { - final ColumnName keyName; - final SqlType keyType; - - if (groupByExps.size() != 1) { - keyType = SqlTypes.STRING; - - keyName = ColumnNames.nextKsqlColAlias( - sourceSchema, - LogicalSchema.builder() - .valueColumns(projectionSchema.value()) - .build() - ); - } else { - final ExpressionTypeManager typeManager = - new ExpressionTypeManager(sourceSchema, metaStore); - - final Expression expression = groupByExps.get(0); - - keyType = typeManager.getExpressionSqlType(expression); - keyName = selectResolver.apply(expression) - .orElseGet(() -> expression instanceof ColumnReferenceExp - ? ((ColumnReferenceExp) expression).getColumnName() - : ColumnNames.uniqueAliasFor(expression, sourceSchema) - ); - } + for (final Expression expression : groupByExps) { + final SqlType keyType = typeManager.getExpressionSqlType(expression); + final ColumnName keyName = selectResolver.apply(expression) + .orElseGet(() -> expression instanceof ColumnReferenceExp + ? ((ColumnReferenceExp) expression).getColumnName() + : ColumnNames.uniqueAliasFor(expression, sourceSchema) + ); builder.keyColumn(keyName, keyType); } diff --git a/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKStream.java b/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKStream.java index 3ce745213721..5a70f784fb98 100644 --- a/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKStream.java +++ b/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKStream.java @@ -28,11 +28,11 @@ import io.confluent.ksql.execution.plan.ExecutionStep; import io.confluent.ksql.execution.plan.Formats; import io.confluent.ksql.execution.plan.JoinType; -import io.confluent.ksql.execution.plan.KGroupedStreamHolder; import io.confluent.ksql.execution.plan.KStreamHolder; import io.confluent.ksql.execution.plan.SelectExpression; import io.confluent.ksql.execution.plan.StreamFilter; import io.confluent.ksql.execution.plan.StreamFlatMap; +import io.confluent.ksql.execution.plan.StreamGroupBy; import io.confluent.ksql.execution.plan.StreamGroupByKey; import io.confluent.ksql.execution.plan.StreamSelect; import io.confluent.ksql.execution.plan.StreamSink; @@ -371,32 +371,16 @@ public SchemaKGroupedStream groupBy( rekeyedFormat = keyFormat; } - final boolean isSingleKey; - if (ksqlConfig.getBoolean(KsqlConfig.KSQL_MULTICOL_KEY_FORMAT_ENABLED)) { - isSingleKey = groupByExpressions.size() == 1; - } else { - isSingleKey = true; - } - final KeyFormat sanitizedKeyFormat = SerdeFeaturesFactory.sanitizeKeyFormat( - rekeyedFormat, isSingleKey); - - final ExecutionStep source; - if (ksqlConfig.getBoolean(KsqlConfig.KSQL_MULTICOL_KEY_FORMAT_ENABLED)) { - source = ExecutionStepFactory.streamGroupBy( - contextStacker, - sourceStep, - InternalFormats.of(sanitizedKeyFormat, valueFormat), - groupByExpressions - ); - } else { - source = ExecutionStepFactory.streamGroupByV1( - contextStacker, - sourceStep, - InternalFormats.of(sanitizedKeyFormat, valueFormat), - groupByExpressions - ); - } + rekeyedFormat, + groupByExpressions.size() == 1 + ); + final StreamGroupBy source = ExecutionStepFactory.streamGroupBy( + contextStacker, + sourceStep, + InternalFormats.of(sanitizedKeyFormat, valueFormat), + groupByExpressions + ); return new SchemaKGroupedStream( source, diff --git a/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKTable.java b/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKTable.java index 69c553c05ae8..8925c7c2b693 100644 --- a/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKTable.java +++ b/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKTable.java @@ -25,10 +25,10 @@ import io.confluent.ksql.execution.plan.ExecutionStep; import io.confluent.ksql.execution.plan.Formats; import io.confluent.ksql.execution.plan.JoinType; -import io.confluent.ksql.execution.plan.KGroupedTableHolder; import io.confluent.ksql.execution.plan.KTableHolder; import io.confluent.ksql.execution.plan.SelectExpression; import io.confluent.ksql.execution.plan.TableFilter; +import io.confluent.ksql.execution.plan.TableGroupBy; import io.confluent.ksql.execution.plan.TableSelect; import io.confluent.ksql.execution.plan.TableSink; import io.confluent.ksql.execution.plan.TableSuppress; @@ -221,37 +221,20 @@ public SchemaKGroupedTable groupBy( final List groupByExpressions, final Stacker contextStacker ) { - final boolean isSingleKey; - if (ksqlConfig.getBoolean(KsqlConfig.KSQL_MULTICOL_KEY_FORMAT_ENABLED)) { - isSingleKey = groupByExpressions.size() == 1; - } else { - isSingleKey = true; - } - // Since tables must have a key, we know that the keyFormat is both // not NONE and has at least one column; this allows us to inherit // the key format directly (as opposed to the logic in SchemaKStream) final KeyFormat groupedKeyFormat = SerdeFeaturesFactory.sanitizeKeyFormat( KeyFormat.nonWindowed(keyFormat.getFormatInfo(), keyFormat.getFeatures()), - isSingleKey + groupByExpressions.size() == 1 ); - final ExecutionStep step; - if (ksqlConfig.getBoolean(KsqlConfig.KSQL_MULTICOL_KEY_FORMAT_ENABLED)) { - step = ExecutionStepFactory.tableGroupBy( - contextStacker, - sourceTableStep, - InternalFormats.of(groupedKeyFormat, valueFormat), - groupByExpressions - ); - } else { - step = ExecutionStepFactory.tableGroupByV1( - contextStacker, - sourceTableStep, - InternalFormats.of(groupedKeyFormat, valueFormat), - groupByExpressions - ); - } + final TableGroupBy step = ExecutionStepFactory.tableGroupBy( + contextStacker, + sourceTableStep, + InternalFormats.of(groupedKeyFormat, valueFormat), + groupByExpressions + ); return new SchemaKGroupedTable( step, diff --git a/ksqldb-engine/src/test/java/io/confluent/ksql/structured/SchemaKStreamTest.java b/ksqldb-engine/src/test/java/io/confluent/ksql/structured/SchemaKStreamTest.java index 6c347d16c121..bd00bd5f6799 100644 --- a/ksqldb-engine/src/test/java/io/confluent/ksql/structured/SchemaKStreamTest.java +++ b/ksqldb-engine/src/test/java/io/confluent/ksql/structured/SchemaKStreamTest.java @@ -395,7 +395,7 @@ public void shouldBuildStepForGroupBy() { assertThat( groupedSchemaKStream.getSourceStep(), equalTo( - ExecutionStepFactory.streamGroupByV1( + ExecutionStepFactory.streamGroupBy( childContextStacker, initialSchemaKStream.getSourceStep(), Formats diff --git a/ksqldb-engine/src/test/java/io/confluent/ksql/structured/SchemaKTableTest.java b/ksqldb-engine/src/test/java/io/confluent/ksql/structured/SchemaKTableTest.java index 2f116efa4e88..f4b5ff0b8902 100644 --- a/ksqldb-engine/src/test/java/io/confluent/ksql/structured/SchemaKTableTest.java +++ b/ksqldb-engine/src/test/java/io/confluent/ksql/structured/SchemaKTableTest.java @@ -558,7 +558,7 @@ public void shouldBuildStepForGroupBy() { assertThat( groupedSchemaKTable.getSourceTableStep(), equalTo( - ExecutionStepFactory.tableGroupByV1( + ExecutionStepFactory.tableGroupBy( childContextStacker, initialSchemaKTable.getSourceTableStep(), Formats.of( @@ -593,13 +593,13 @@ public void shouldBuildStepForGroupByBasedOnKeySerdeFeatures() { assertThat( groupedSchemaKTable.getSourceTableStep(), equalTo( - ExecutionStepFactory.tableGroupByV1( + ExecutionStepFactory.tableGroupBy( childContextStacker, initialSchemaKTable.getSourceTableStep(), Formats.of( initialSchemaKTable.keyFormat.getFormatInfo(), valueFormat.getFormatInfo(), - SerdeFeatures.of(SerdeFeature.UNWRAP_SINGLES), + SerdeFeatures.of(), SerdeFeatures.of() ), groupByExpressions diff --git a/ksqldb-functional-tests/src/test/java/io/confluent/ksql/test/planned/PlannedTestUtils.java b/ksqldb-functional-tests/src/test/java/io/confluent/ksql/test/planned/PlannedTestUtils.java index 4e2da6d6a54c..a3baa393ae41 100644 --- a/ksqldb-functional-tests/src/test/java/io/confluent/ksql/test/planned/PlannedTestUtils.java +++ b/ksqldb-functional-tests/src/test/java/io/confluent/ksql/test/planned/PlannedTestUtils.java @@ -24,7 +24,6 @@ import io.confluent.ksql.test.tools.TestCase; import io.confluent.ksql.test.tools.TestCaseBuilder; import io.confluent.ksql.test.tools.TopologyAndConfigs; -import io.confluent.ksql.util.KsqlConfig; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -32,7 +31,6 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Optional; public final class PlannedTestUtils { @@ -49,8 +47,7 @@ public static boolean isPlannedTestCase(final TestCase testCase) { public static boolean isNotExcluded(final TestCase testCase) { // Place temporary logic here to exclude test cases based on feature flags, etc. - final Map props = testCase.properties(); - return !(boolean) props.getOrDefault(KsqlConfig.KSQL_MULTICOL_KEY_FORMAT_ENABLED, false); + return true; } public static boolean isSamePlan( diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_group_by/6.2.0_1608159537782/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_group_by/6.2.0_1608159537782/plan.json new file mode 100644 index 000000000000..b7d473d9c9f9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_group_by/6.2.0_1608159537782/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID INTEGER KEY, V0 INTEGER, V1 INTEGER) WITH (KAFKA_TOPIC='input', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER", + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n AS_VALUE(INPUT.ID) K,\n COUNT(1) KSQL_COL_0\nFROM INPUT INPUT\nGROUP BY AS_VALUE(INPUT.ID)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "1 AS KSQL_INTERNAL_COL_1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "AS_VALUE(ID)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID" ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_1)" ] + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_group_by/6.2.0_1608159537782/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_group_by/6.2.0_1608159537782/spec.json new file mode 100644 index 000000000000..2a6b79590c08 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_group_by/6.2.0_1608159537782/spec.json @@ -0,0 +1,131 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159537782, + "path" : "query-validation-tests/as_value.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `ID` INTEGER, `KSQL_INTERNAL_COL_1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `ID` INTEGER, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "group by", + "inputs" : [ { + "topic" : "input", + "key" : 1, + "value" : { + "V0" : 2, + "V1" : 3 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "input", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (ID INT KEY, V0 INT, V1 INT) WITH (kafka_topic='input', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT AS_VALUE(ID) AS K, COUNT(1) FROM INPUT GROUP BY AS_VALUE(ID);" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER, `V1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "input", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_group_by/6.2.0_1608159537782/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_group_by/6.2.0_1608159537782/topology new file mode 100644 index 000000000000..efaf31bc5c28 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/as_value_-_group_by/6.2.0_1608159537782/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.2.0_1608159547570/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.2.0_1608159547570/plan.json new file mode 100644 index 000000000000..bd395407d820 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.2.0_1608159547570/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE MAP) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE['key1']) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "VALUE['key1'] AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(KSQL_INTERNAL_COL_2)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.2.0_1608159547570/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.2.0_1608159547570/spec.json new file mode 100644 index 000000000000..63f75885c1fc --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.2.0_1608159547570/spec.json @@ -0,0 +1,186 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159547570, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_INTERNAL_COL_2` BOOLEAN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "collect_list bool map table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : false + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : false, + "key2" : true + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : true + } + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ false ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.2.0_1608159547570/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.2.0_1608159547570/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_JSON/6.2.0_1608159547570/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.2.0_1608159547715/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.2.0_1608159547715/plan.json new file mode 100644 index 000000000000..687399e14518 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.2.0_1608159547715/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE MAP) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE['key1']) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "VALUE['key1'] AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(KSQL_INTERNAL_COL_2)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.2.0_1608159547715/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.2.0_1608159547715/spec.json new file mode 100644 index 000000000000..a47533577e05 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.2.0_1608159547715/spec.json @@ -0,0 +1,193 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159547715, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_INTERNAL_COL_2` BOOLEAN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + } + }, + "testCase" : { + "name" : "collect_list bool map table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : false + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : false, + "key2" : true + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : true + } + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ false ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n repeated ConnectDefault2Entry VALUE = 2;\n\n message ConnectDefault2Entry {\n string key = 1;\n bool value = 2;\n }\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n repeated ConnectDefault2Entry VALUE = 2;\n repeated bool KSQL_AGG_VARIABLE_0 = 3;\n\n message ConnectDefault2Entry {\n string key = 1;\n bool value = 2;\n }\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n repeated ConnectDefault2Entry VALUE = 2;\n\n message ConnectDefault2Entry {\n string key = 1;\n bool value = 2;\n }\n}\n" + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n repeated ConnectDefault2Entry VALUE = 2;\n\n message ConnectDefault2Entry {\n string key = 1;\n bool value = 2;\n }\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n repeated ConnectDefault2Entry VALUE = 2;\n bool KSQL_INTERNAL_COL_2 = 3;\n\n message ConnectDefault2Entry {\n string key = 1;\n bool value = 2;\n }\n}\n" + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n repeated bool COLLECTED = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.2.0_1608159547715/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.2.0_1608159547715/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF/6.2.0_1608159547715/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.2.0_1608159546621/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.2.0_1608159546621/plan.json new file mode 100644 index 000000000000..9b973cc1cf5a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.2.0_1608159546621/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE DOUBLE) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.2.0_1608159546621/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.2.0_1608159546621/spec.json new file mode 100644 index 000000000000..f90d4b362060 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.2.0_1608159546621/spec.json @@ -0,0 +1,267 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159546621, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` DOUBLE, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "collect_list double table - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 5.4 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100.1 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500.9 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 300.8 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 5.4 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100.1 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500.9 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 300.8 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "double" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE double) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "double" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", { + "type" : "array", + "items" : [ "null", "double" ] + } ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "double" ], + "default" : null + } ] + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "double" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "double" ], + "default" : null + } ] + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "COLLECTED", + "type" : [ "null", { + "type" : "array", + "items" : [ "null", "double" ] + } ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.2.0_1608159546621/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.2.0_1608159546621/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_AVRO/6.2.0_1608159546621/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.2.0_1608159546749/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.2.0_1608159546749/plan.json new file mode 100644 index 000000000000..2318cece773f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.2.0_1608159546749/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE DOUBLE) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.2.0_1608159546749/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.2.0_1608159546749/spec.json new file mode 100644 index 000000000000..d7ec55b9f071 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.2.0_1608159546749/spec.json @@ -0,0 +1,186 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159546749, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` DOUBLE, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "collect_list double table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 5.4 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100.1 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500.9 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 300.8 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 5.4 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100.1 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500.9 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 300.8 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE double) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.2.0_1608159546749/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.2.0_1608159546749/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_JSON/6.2.0_1608159546749/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.2.0_1608159547037/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.2.0_1608159547037/plan.json new file mode 100644 index 000000000000..36302a1fe598 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.2.0_1608159547037/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE DOUBLE) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.2.0_1608159547037/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.2.0_1608159547037/spec.json new file mode 100644 index 000000000000..7787161b1433 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.2.0_1608159547037/spec.json @@ -0,0 +1,193 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159547037, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` DOUBLE, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + } + }, + "testCase" : { + "name" : "collect_list double table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 5.4 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100.1 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500.9 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 300.8 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 5.4 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100.1 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500.9 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 300.8 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n double VALUE = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE double) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n double VALUE = 2;\n repeated double KSQL_AGG_VARIABLE_0 = 3;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n double VALUE = 1;\n}\n" + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n double VALUE = 1;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n double VALUE = 2;\n}\n" + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n repeated double COLLECTED = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.2.0_1608159547037/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.2.0_1608159547037/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF/6.2.0_1608159547037/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.2.0_1608159545729/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.2.0_1608159545729/plan.json new file mode 100644 index 000000000000..e4fd2fe7a46d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.2.0_1608159545729/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.2.0_1608159545729/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.2.0_1608159545729/spec.json new file mode 100644 index 000000000000..c4712b17bbc6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.2.0_1608159545729/spec.json @@ -0,0 +1,267 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159545729, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "collect_list int table - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 0 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE integer) WITH (kafka_topic='test_topic',value_format='AVRO');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", { + "type" : "array", + "items" : [ "null", "int" ] + } ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "int" ], + "default" : null + } ] + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "int" ], + "default" : null + } ] + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "COLLECTED", + "type" : [ "null", { + "type" : "array", + "items" : [ "null", "int" ] + } ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.2.0_1608159545729/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.2.0_1608159545729/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_AVRO/6.2.0_1608159545729/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.2.0_1608159545912/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.2.0_1608159545912/plan.json new file mode 100644 index 000000000000..e4efe62c4052 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.2.0_1608159545912/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.2.0_1608159545912/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.2.0_1608159545912/spec.json new file mode 100644 index 000000000000..07e8dc7bb64b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.2.0_1608159545912/spec.json @@ -0,0 +1,186 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159545912, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "collect_list int table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 0 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE integer) WITH (kafka_topic='test_topic',value_format='JSON');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.2.0_1608159545912/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.2.0_1608159545912/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_JSON/6.2.0_1608159545912/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.2.0_1608159546057/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.2.0_1608159546057/plan.json new file mode 100644 index 000000000000..fb0d2c8f80b2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.2.0_1608159546057/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.2.0_1608159546057/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.2.0_1608159546057/spec.json new file mode 100644 index 000000000000..776265ea11cf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.2.0_1608159546057/spec.json @@ -0,0 +1,193 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159546057, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + } + }, + "testCase" : { + "name" : "collect_list int table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 0 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 VALUE = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE integer) WITH (kafka_topic='test_topic',value_format='PROTOBUF');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n int32 VALUE = 2;\n repeated int32 KSQL_AGG_VARIABLE_0 = 3;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 VALUE = 1;\n}\n" + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 VALUE = 1;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n int32 VALUE = 2;\n}\n" + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n repeated int32 COLLECTED = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.2.0_1608159546057/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.2.0_1608159546057/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF/6.2.0_1608159546057/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.2.0_1608159546201/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.2.0_1608159546201/plan.json new file mode 100644 index 000000000000..8da62a5655f7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.2.0_1608159546201/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.2.0_1608159546201/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.2.0_1608159546201/spec.json new file mode 100644 index 000000000000..72b86618b36b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.2.0_1608159546201/spec.json @@ -0,0 +1,267 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159546201, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "collect_list long table - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 2147483648 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 2147483648 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE bigint) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", { + "type" : "array", + "items" : [ "null", "long" ] + } ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "COLLECTED", + "type" : [ "null", { + "type" : "array", + "items" : [ "null", "long" ] + } ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.2.0_1608159546201/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.2.0_1608159546201/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_AVRO/6.2.0_1608159546201/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.2.0_1608159546338/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.2.0_1608159546338/plan.json new file mode 100644 index 000000000000..deb8d84f673a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.2.0_1608159546338/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.2.0_1608159546338/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.2.0_1608159546338/spec.json new file mode 100644 index 000000000000..7a02d1cfac32 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.2.0_1608159546338/spec.json @@ -0,0 +1,186 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159546338, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "collect_list long table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 2147483648 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 2147483648 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE bigint) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.2.0_1608159546338/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.2.0_1608159546338/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_JSON/6.2.0_1608159546338/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.2.0_1608159546456/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.2.0_1608159546456/plan.json new file mode 100644 index 000000000000..5317ada046b1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.2.0_1608159546456/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.2.0_1608159546456/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.2.0_1608159546456/spec.json new file mode 100644 index 000000000000..c39709c79973 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.2.0_1608159546456/spec.json @@ -0,0 +1,193 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159546456, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + } + }, + "testCase" : { + "name" : "collect_list long table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 2147483648 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 2147483648 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 VALUE = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE bigint) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n int64 VALUE = 2;\n repeated int64 KSQL_AGG_VARIABLE_0 = 3;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 VALUE = 1;\n}\n" + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 VALUE = 1;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n int64 VALUE = 2;\n}\n" + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n repeated int64 COLLECTED = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.2.0_1608159546456/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.2.0_1608159546456/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF/6.2.0_1608159546456/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.2.0_1608159547170/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.2.0_1608159547170/plan.json new file mode 100644 index 000000000000..999b4f97fb83 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.2.0_1608159547170/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.2.0_1608159547170/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.2.0_1608159547170/spec.json new file mode 100644 index 000000000000..c87ae144014f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.2.0_1608159547170/spec.json @@ -0,0 +1,285 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159547170, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "collect_list string table - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "foo" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "bar" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "foo" + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "foo" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "baz" ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "bar" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "baz" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "foo" ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE varchar) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", { + "type" : "array", + "items" : [ "null", "string" ] + } ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "string" ], + "default" : null + } ] + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VALUE", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "VALUE", + "type" : [ "null", "string" ], + "default" : null + } ] + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "COLLECTED", + "type" : [ "null", { + "type" : "array", + "items" : [ "null", "string" ] + } ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.2.0_1608159547170/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.2.0_1608159547170/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_AVRO/6.2.0_1608159547170/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.2.0_1608159547296/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.2.0_1608159547296/plan.json new file mode 100644 index 000000000000..a7e1d498a32c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.2.0_1608159547296/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.2.0_1608159547296/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.2.0_1608159547296/spec.json new file mode 100644 index 000000000000..8d8c347a3ed9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.2.0_1608159547296/spec.json @@ -0,0 +1,204 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159547296, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "collect_list string table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "foo" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "bar" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "foo" + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "foo" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "baz" ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "bar" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "baz" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "foo" ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE varchar) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.2.0_1608159547296/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.2.0_1608159547296/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_JSON/6.2.0_1608159547296/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.2.0_1608159547424/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.2.0_1608159547424/plan.json new file mode 100644 index 000000000000..4b02af4c663b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.2.0_1608159547424/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.2.0_1608159547424/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.2.0_1608159547424/spec.json new file mode 100644 index 000000000000..3aa137d55455 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.2.0_1608159547424/spec.json @@ -0,0 +1,211 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159547424, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + } + }, + "testCase" : { + "name" : "collect_list string table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "foo" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "bar" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "foo" + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "foo" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "baz" ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "bar" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "baz" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "foo" ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string VALUE = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE varchar) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n string VALUE = 2;\n repeated string KSQL_AGG_VARIABLE_0 = 3;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string VALUE = 1;\n}\n" + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string VALUE = 1;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n string VALUE = 2;\n}\n" + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n repeated string COLLECTED = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.2.0_1608159547424/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.2.0_1608159547424/topology new file mode 100644 index 000000000000..02c21d6d3fa1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF/6.2.0_1608159547424/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count-distinct_-_count_distinct/6.2.0_1608159549589/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count-distinct_-_count_distinct/6.2.0_1608159549589/plan.json new file mode 100644 index 000000000000..5e9f372d0775 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count-distinct_-_count_distinct/6.2.0_1608159549589/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, ID STRING, NAME STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` STRING, `NAME` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COUNT_DISTINCT(TEST.NAME) COUNT\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` STRING KEY, `COUNT` BIGINT", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` STRING, `NAME` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "NAME AS NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "NAME" ], + "aggregationFunctions" : [ "COUNT_DISTINCT(NAME)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count-distinct_-_count_distinct/6.2.0_1608159549589/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count-distinct_-_count_distinct/6.2.0_1608159549589/spec.json new file mode 100644 index 000000000000..fab69b70f33f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count-distinct_-_count_distinct/6.2.0_1608159549589/spec.json @@ -0,0 +1,196 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159549589, + "path" : "query-validation-tests/count-distinct.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` STRING KEY, `ID` STRING, `NAME` STRING, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ID` STRING, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` STRING KEY, `ID` STRING, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "count distinct", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "id" : "foo", + "name" : "one" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "id" : "foo", + "name" : "two" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "id" : "foo", + "name" : "one" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "id" : "foo", + "name" : "two" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "id" : "bar", + "name" : "one" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "id" : "foo", + "name" : null + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : "foo", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "S2", + "key" : "foo", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "S2", + "key" : "foo", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "S2", + "key" : "foo", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "S2", + "key" : "bar", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "S2", + "key" : "foo", + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, ID varchar, NAME varchar) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE S2 as SELECT ID, count_distinct(name) as count FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` STRING, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count-distinct_-_count_distinct/6.2.0_1608159549589/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/count-distinct_-_count_distinct/6.2.0_1608159549589/topology new file mode 100644 index 000000000000..dffbdbae1c71 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count-distinct_-_count_distinct/6.2.0_1608159549589/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: S2) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_auto-incrementing_id/6.2.0_1608159549507/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_auto-incrementing_id/6.2.0_1608159549507/plan.json new file mode 100644 index 000000000000..eb754f0d58cb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_auto-incrementing_id/6.2.0_1608159549507/plan.json @@ -0,0 +1,180 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n 1 K,\n LATEST_BY_OFFSET(INPUT.VALUE) VALUE,\n COUNT(1) ID\nFROM INPUT INPUT\nGROUP BY 1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `VALUE` INTEGER, `ID` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`VALUE` INTEGER" + }, + "selectExpressions" : [ "VALUE AS VALUE", "1 AS KSQL_INTERNAL_COL_1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "VALUE" ], + "aggregationFunctions" : [ "LATEST_BY_OFFSET(VALUE)", "COUNT(KSQL_INTERNAL_COL_1)" ] + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS VALUE", "KSQL_AGG_VARIABLE_1 AS ID" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_auto-incrementing_id/6.2.0_1608159549507/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_auto-incrementing_id/6.2.0_1608159549507/spec.json new file mode 100644 index 000000000000..5a837935d43e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_auto-incrementing_id/6.2.0_1608159549507/spec.json @@ -0,0 +1,157 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159549507, + "path" : "query-validation-tests/count.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `VALUE` INTEGER, `KSQL_INTERNAL_COL_1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `VALUE` INTEGER, `KSQL_AGG_VARIABLE_0` STRUCT<`SEQ` BIGINT, `VAL` INTEGER>, `KSQL_AGG_VARIABLE_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `VALUE` INTEGER, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "auto-incrementing id", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "VALUE" : 12 + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "VALUE" : 8367 + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "VALUE" : 764 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "VALUE" : 12, + "ID" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "VALUE" : 8367, + "ID" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "VALUE" : 764, + "ID" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (VALUE INT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT 1 as k, latest_by_offset(value) as value, count(1) AS ID FROM INPUT group by 1;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `VALUE` INTEGER, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_auto-incrementing_id/6.2.0_1608159549507/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_auto-incrementing_id/6.2.0_1608159549507/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_auto-incrementing_id/6.2.0_1608159549507/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.2.0_1608159549188/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.2.0_1608159549188/plan.json new file mode 100644 index 000000000000..966fdcd89747 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.2.0_1608159549188/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (ID STRING PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='input_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`ID` STRING KEY, `NAME` STRING", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.NAME NAME,\n COUNT(1) KSQL_COL_0\nFROM INPUT INPUT\nGROUP BY INPUT.NAME\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NAME` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` STRING KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "NAME AS NAME", "1 AS KSQL_INTERNAL_COL_1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "NAME" ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_1)" ] + }, + "keyColumnNames" : [ "NAME" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.2.0_1608159549188/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.2.0_1608159549188/spec.json new file mode 100644 index 000000000000..33a0125197d5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.2.0_1608159549188/spec.json @@ -0,0 +1,162 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159549188, + "path" : "query-validation-tests/count.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`NAME` STRING KEY, `NAME` STRING, `KSQL_INTERNAL_COL_1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` STRING KEY, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`NAME` STRING KEY, `NAME` STRING, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NAME` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "count table", + "inputs" : [ { + "topic" : "input_topic", + "key" : "0", + "value" : "bob" + }, { + "topic" : "input_topic", + "key" : "0", + "value" : "john" + }, { + "topic" : "input_topic", + "key" : "100", + "value" : "john" + }, { + "topic" : "input_topic", + "key" : "100", + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "bob", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "bob", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "john", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "john", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "john", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (ID STRING PRIMARY KEY, name STRING) WITH (kafka_topic='input_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT as SELECT NAME, count(1) FROM input group by name;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "TABLE", + "schema" : "`ID` STRING KEY, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NAME` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.2.0_1608159549188/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.2.0_1608159549188/topology new file mode 100644 index 000000000000..6e6a85aa7919 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_count_table/6.2.0_1608159549188/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [input_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.2.0_1608159549310/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.2.0_1608159549310/plan.json new file mode 100644 index 000000000000..f2a00cd4b0cc --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.2.0_1608159549310/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID BIGINT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`K` STRING KEY, `ID` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.ID ID,\n COUNT(*) KSQL_COL_0\nFROM INPUT INPUT\nGROUP BY INPUT.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ID` BIGINT KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ID", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.2.0_1608159549310/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.2.0_1608159549310/spec.json new file mode 100644 index 000000000000..4e4f3ece0bb0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.2.0_1608159549310/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159549310, + "path" : "query-validation-tests/count.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ID` BIGINT KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "should count back to zero", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "3" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "3" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "0" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID bigint) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT as SELECT ID, COUNT() FROM INPUT GROUP BY ID;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.2.0_1608159549310/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.2.0_1608159549310/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_count_back_to_zero/6.2.0_1608159549310/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.2.0_1608159549409/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.2.0_1608159549409/plan.json new file mode 100644 index 000000000000..103ca5f0e6c3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.2.0_1608159549409/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID BIGINT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`K` STRING KEY, `ID` BIGINT", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.ID ID,\n COUNT(*) KSQL_COL_0\nFROM INPUT INPUT\nGROUP BY INPUT.ID\nHAVING (COUNT(*) > 0)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ID` BIGINT KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ID", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)", "COUNT(ROWTIME)" ] + }, + "filterExpression" : "(KSQL_AGG_VARIABLE_1 > 0)" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.2.0_1608159549409/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.2.0_1608159549409/spec.json new file mode 100644 index 000000000000..590374baaa44 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.2.0_1608159549409/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159549409, + "path" : "query-validation-tests/count.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT, `KSQL_AGG_VARIABLE_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ID` BIGINT KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "should support removing zero counts from table", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "3" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "3" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : null + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (K STRING PRIMARY KEY, ID bigint) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT as SELECT ID, COUNT() FROM INPUT GROUP BY ID HAVING COUNT() > 0;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.2.0_1608159549409/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.2.0_1608159549409/topology new file mode 100644 index 000000000000..3a244cb18785 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/count_-_should_support_removing_zero_counts_from_table/6.2.0_1608159549409/topology @@ -0,0 +1,52 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_group_by_struct/6.2.0_1608159568672/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_group_by_struct/6.2.0_1608159568672/plan.json new file mode 100644 index 000000000000..d328017093d8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_group_by_struct/6.2.0_1608159568672/plan.json @@ -0,0 +1,205 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, A INTEGER, B INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n STRUCT(A:=TEST.A, B:=TEST.B) ROWKEY,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY STRUCT(A:=TEST.A, B:=TEST.B)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "A AS A", "B AS B", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "STRUCT(A:=A, B:=B)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "A", "B", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_group_by_struct/6.2.0_1608159568672/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_group_by_struct/6.2.0_1608159568672/spec.json new file mode 100644 index 000000000000..08b431ff290f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_group_by_struct/6.2.0_1608159568672/spec.json @@ -0,0 +1,317 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159568672, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `A` INTEGER, `B` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `A` INTEGER, `B` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "AVRO group by struct", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "A" : 1, + "B" : 1 + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "A" : 2, + "B" : 1 + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "A" : 1, + "B" : 1 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "A" : 1, + "B" : 1 + }, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "A" : 2, + "B" : 1 + }, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "A" : 1, + "B" : 1 + }, + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "ignored", + "fields" : [ { + "name" : "A", + "type" : "int" + }, { + "name" : "B", + "type" : "int" + } ] + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, A INT, B INT) WITH (kafka_topic='test_topic', format='AVRO');", "CREATE TABLE OUTPUT AS SELECT STRUCT(a:=A, b:=B) AS ROWKEY, COUNT() AS COUNT FROM TEST GROUP BY STRUCT(a:=A, b:=B);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`ROWKEY` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 1, + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "ignored", + "fields" : [ { + "name" : "A", + "type" : "int" + }, { + "name" : "B", + "type" : "int" + } ], + "connect.name" : "ignored" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keySchema" : { + "type" : "record", + "name" : "TestKey", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "A", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "B", + "type" : [ "null", "int" ], + "default" : null + } ] + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "A", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "B", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keySchema" : { + "type" : "record", + "name" : "TestKey", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "A", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "B", + "type" : [ "null", "int" ], + "default" : null + } ] + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "A", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "B", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : { + "type" : "record", + "name" : "OutputKey", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "A", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "B", + "type" : [ "null", "int" ], + "default" : null + } ] + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "COUNT", + "type" : [ "null", "long" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_group_by_struct/6.2.0_1608159568672/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_group_by_struct/6.2.0_1608159568672/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_group_by_struct/6.2.0_1608159568672/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1608159568495/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1608159568495/plan.json new file mode 100644 index 000000000000..0badc14bdd83 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1608159568495/plan.json @@ -0,0 +1,205 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, VAL BOOLEAN) WITH (FORMAT='AVRO', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `VAL` BOOLEAN", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.VAL VAL,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.VAL\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `VAL` BOOLEAN" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "VAL AS VAL", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "VAL" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "VAL", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "VAL" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1608159568495/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1608159568495/spec.json new file mode 100644 index 000000000000..d197015199d4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1608159568495/spec.json @@ -0,0 +1,252 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159568495, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`VAL` BOOLEAN KEY, `VAL` BOOLEAN, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `VAL` BOOLEAN", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`VAL` BOOLEAN KEY, `VAL` BOOLEAN, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "AVRO primitive key", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VAL" : true + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "VAL" : false + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "VAL" : true + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : false, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "ignored", + "fields" : [ { + "name" : "VAL", + "type" : "boolean" + } ] + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, VAL BOOLEAN) WITH (kafka_topic='test_topic', format='AVRO');", "CREATE TABLE OUTPUT AS SELECT VAL, COUNT() AS COUNT FROM TEST GROUP BY VAL;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `VAL` BOOLEAN", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 1, + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "ignored", + "fields" : [ { + "name" : "VAL", + "type" : "boolean" + } ], + "connect.name" : "ignored" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VAL", + "type" : [ "null", "boolean" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VAL", + "type" : [ "null", "boolean" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "COUNT", + "type" : [ "null", "long" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1608159568495/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1608159568495/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1608159568495/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_struct_key_group_by_primitive/6.2.0_1608159568578/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_struct_key_group_by_primitive/6.2.0_1608159568578/plan.json new file mode 100644 index 000000000000..d3692f51b2aa --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_struct_key_group_by_primitive/6.2.0_1608159568578/plan.json @@ -0,0 +1,205 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID STRUCT KEY, VAL BOOLEAN) WITH (FORMAT='AVRO', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` STRUCT<`F1` INTEGER> KEY, `VAL` BOOLEAN", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.VAL VAL,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.VAL\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` STRUCT<`F1` INTEGER> KEY, `VAL` BOOLEAN" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "VAL AS VAL", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "VAL" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "VAL", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "VAL" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_struct_key_group_by_primitive/6.2.0_1608159568578/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_struct_key_group_by_primitive/6.2.0_1608159568578/spec.json new file mode 100644 index 000000000000..23d43f4dd55a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_struct_key_group_by_primitive/6.2.0_1608159568578/spec.json @@ -0,0 +1,274 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159568578, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`VAL` BOOLEAN KEY, `VAL` BOOLEAN, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` STRUCT<`F1` INTEGER> KEY, `VAL` BOOLEAN", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`VAL` BOOLEAN KEY, `VAL` BOOLEAN, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "AVRO struct key group by primitive", + "inputs" : [ { + "topic" : "test_topic", + "key" : { + "F1" : 0 + }, + "value" : { + "VAL" : true + } + }, { + "topic" : "test_topic", + "key" : { + "F1" : 0 + }, + "value" : { + "VAL" : false + } + }, { + "topic" : "test_topic", + "key" : { + "F1" : 1 + }, + "value" : { + "VAL" : true + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : false, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "keySchema" : { + "type" : "record", + "name" : "key", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ] + } ] + }, + "valueSchema" : { + "type" : "record", + "name" : "ignored", + "fields" : [ { + "name" : "VAL", + "type" : "boolean" + } ] + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM TEST (ID STRUCT KEY, VAL BOOLEAN) WITH (kafka_topic='test_topic', format='AVRO');", "CREATE TABLE OUTPUT AS SELECT VAL, COUNT() AS COUNT FROM TEST GROUP BY VAL;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` STRUCT<`F1` INTEGER> KEY, `VAL` BOOLEAN", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 1, + "keySchema" : { + "type" : "record", + "name" : "key", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "key" + }, + "valueSchema" : { + "type" : "record", + "name" : "ignored", + "fields" : [ { + "name" : "VAL", + "type" : "boolean" + } ], + "connect.name" : "ignored" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VAL", + "type" : [ "null", "boolean" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.TestKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VAL", + "type" : [ "null", "boolean" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "COUNT", + "type" : [ "null", "long" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_struct_key_group_by_primitive/6.2.0_1608159568578/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_struct_key_group_by_primitive/6.2.0_1608159568578/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_struct_key_group_by_primitive/6.2.0_1608159568578/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_array/6.2.0_1608159568752/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_array/6.2.0_1608159568752/plan.json new file mode 100644 index 000000000000..3647b5d1303b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_array/6.2.0_1608159568752/plan.json @@ -0,0 +1,187 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, A INTEGER, B INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n ARRAY[TEST.A, TEST.B] ROWKEY,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY ARRAY[TEST.A, TEST.B]\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` ARRAY KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "A AS A", "B AS B", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "ARRAY[A, B]" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "A", "B", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_array/6.2.0_1608159568752/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_array/6.2.0_1608159568752/spec.json new file mode 100644 index 000000000000..383372469aa4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_array/6.2.0_1608159568752/spec.json @@ -0,0 +1,165 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159568752, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` ARRAY KEY, `A` INTEGER, `B` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` ARRAY KEY, `A` INTEGER, `B` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` ARRAY KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "JSON group by array", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "A" : 1, + "B" : 1 + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "A" : 2, + "B" : 1 + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "A" : 1, + "B" : 1 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : [ 1, 1 ], + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : [ 2, 1 ], + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : [ 1, 1 ], + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, A INT, B INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT ARRAY[a, b] AS ROWKEY, COUNT() AS COUNT FROM TEST GROUP BY ARRAY[A, B];" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`ROWKEY` ARRAY KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_array/6.2.0_1608159568752/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_array/6.2.0_1608159568752/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_array/6.2.0_1608159568752/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct/6.2.0_1608159568815/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct/6.2.0_1608159568815/plan.json new file mode 100644 index 000000000000..2b18d51f33a0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct/6.2.0_1608159568815/plan.json @@ -0,0 +1,187 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, A INTEGER, B INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n STRUCT(A:=TEST.A, B:=TEST.B) ROWKEY,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY STRUCT(A:=TEST.A, B:=TEST.B)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "A AS A", "B AS B", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "STRUCT(A:=A, B:=B)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "A", "B", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct/6.2.0_1608159568815/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct/6.2.0_1608159568815/spec.json new file mode 100644 index 000000000000..3d02510ec810 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct/6.2.0_1608159568815/spec.json @@ -0,0 +1,174 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159568815, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `A` INTEGER, `B` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `A` INTEGER, `B` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "JSON group by struct", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "A" : 1, + "B" : 1 + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "A" : 2, + "B" : 1 + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "A" : 1, + "B" : 1 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "A" : 1, + "B" : 1 + }, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "A" : 2, + "B" : 1 + }, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "A" : 1, + "B" : 1 + }, + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, A INT, B INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT STRUCT(a:=A, b:=B) AS ROWKEY, COUNT() AS COUNT FROM TEST GROUP BY STRUCT(a:=A, b:=B);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`ROWKEY` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct/6.2.0_1608159568815/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct/6.2.0_1608159568815/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct/6.2.0_1608159568815/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct_convert_key_format/6.2.0_1608159568888/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct_convert_key_format/6.2.0_1608159568888/plan.json new file mode 100644 index 000000000000..f1d30ae2ef63 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct_convert_key_format/6.2.0_1608159568888/plan.json @@ -0,0 +1,193 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, A INTEGER, B INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT WITH (KEY_FORMAT='AVRO') AS SELECT\n STRUCT(A:=TEST.A, B:=TEST.B) ROWKEY,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY STRUCT(A:=TEST.A, B:=TEST.B)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "A AS A", "B AS B", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "STRUCT(A:=A, B:=B)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "A", "B", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct_convert_key_format/6.2.0_1608159568888/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct_convert_key_format/6.2.0_1608159568888/spec.json new file mode 100644 index 000000000000..43ed05a95272 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct_convert_key_format/6.2.0_1608159568888/spec.json @@ -0,0 +1,194 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159568888, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `A` INTEGER, `B` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `A` INTEGER, `B` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "JSON group by struct convert key format", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "A" : 1, + "B" : 1 + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "A" : 2, + "B" : 1 + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "A" : 1, + "B" : 1 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "A" : 1, + "B" : 1 + }, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "A" : 2, + "B" : 1 + }, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "A" : 1, + "B" : 1 + }, + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, A INT, B INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT WITH (key_format='AVRO') AS SELECT STRUCT(a:=A, b:=B) AS ROWKEY, COUNT() AS COUNT FROM TEST GROUP BY STRUCT(a:=A, b:=B);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`ROWKEY` STRUCT<`A` INTEGER, `B` INTEGER> KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `A` INTEGER, `B` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "type" : "record", + "name" : "OutputKey", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "A", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "B", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct_convert_key_format/6.2.0_1608159568888/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct_convert_key_format/6.2.0_1608159568888/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_JSON_group_by_struct_convert_key_format/6.2.0_1608159568888/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_ROWTIME_(stream-_table)/6.2.0_1608159564405/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_ROWTIME_(stream-_table)/6.2.0_1608159564405/plan.json new file mode 100644 index 000000000000..288dd2b92aaf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_ROWTIME_(stream-_table)/6.2.0_1608159564405/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, IGNORED STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `IGNORED` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.ROWTIME RT,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.ROWTIME\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`RT` BIGINT KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `IGNORED` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "RT" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_ROWTIME_(stream-_table)/6.2.0_1608159564405/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_ROWTIME_(stream-_table)/6.2.0_1608159564405/spec.json new file mode 100644 index 000000000000..da97c52c3712 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_ROWTIME_(stream-_table)/6.2.0_1608159564405/spec.json @@ -0,0 +1,128 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159564405, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`ROWTIME` BIGINT KEY, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `IGNORED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ROWTIME` BIGINT KEY, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`RT` BIGINT KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "ROWTIME (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "-", + "timestamp" : 10 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : "1", + "timestamp" : 10 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, ignored VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT ROWTIME AS RT, COUNT(*) FROM TEST GROUP BY ROWTIME;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`RT` BIGINT KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `IGNORED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_ROWTIME_(stream-_table)/6.2.0_1608159564405/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_ROWTIME_(stream-_table)/6.2.0_1608159564405/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_ROWTIME_(stream-_table)/6.2.0_1608159564405/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(stream-_table)/6.2.0_1608159566489/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(stream-_table)/6.2.0_1608159566489/plan.json new file mode 100644 index 000000000000..31745ad27423 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(stream-_table)/6.2.0_1608159566489/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, D1 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `D1` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.D1 D1,\n SUBSTRING('Mr Bugalicious', CAST(COUNT(*) AS INTEGER), 1) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.D1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`D1` STRING KEY, `KSQL_COL_0` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `D1` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "D1 AS D1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "D1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "D1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "D1" ], + "selectExpressions" : [ "SUBSTRING('Mr Bugalicious', CAST(KSQL_AGG_VARIABLE_0 AS INTEGER), 1) AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(stream-_table)/6.2.0_1608159566489/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(stream-_table)/6.2.0_1608159566489/spec.json new file mode 100644 index 000000000000..08d1dbc38886 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(stream-_table)/6.2.0_1608159566489/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159566489, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`D1` STRING KEY, `D1` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `D1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`D1` STRING KEY, `D1` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`D1` STRING KEY, `KSQL_COL_0` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "UDAF nested in UDF in select expression (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "x" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "xxx" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "y" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "x" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "xxx" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "x", + "value" : "M" + }, { + "topic" : "OUTPUT", + "key" : "xxx", + "value" : "M" + }, { + "topic" : "OUTPUT", + "key" : "y", + "value" : "M" + }, { + "topic" : "OUTPUT", + "key" : "x", + "value" : "r" + }, { + "topic" : "OUTPUT", + "key" : "xxx", + "value" : "r" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, d1 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT D1, SUBSTRING('Mr Bugalicious', CAST(COUNT(*) AS INT), 1) FROM TEST GROUP BY d1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`D1` STRING KEY, `KSQL_COL_0` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `D1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(stream-_table)/6.2.0_1608159566489/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(stream-_table)/6.2.0_1608159566489/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(stream-_table)/6.2.0_1608159566489/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.2.0_1608159566596/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.2.0_1608159566596/plan.json new file mode 100644 index 000000000000..04729c5d81c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.2.0_1608159566596/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, D0 INTEGER, D1 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `D0` INTEGER, `D1` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.D1 D1,\n SUBSTRING('Mr Bugalicious', CAST(COUNT(*) AS INTEGER), 1) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.D1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`D1` STRING KEY, `KSQL_COL_0` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `D0` INTEGER, `D1` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "D1 AS D1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "D1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "D1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "D1" ], + "selectExpressions" : [ "SUBSTRING('Mr Bugalicious', CAST(KSQL_AGG_VARIABLE_0 AS INTEGER), 1) AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.2.0_1608159566596/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.2.0_1608159566596/spec.json new file mode 100644 index 000000000000..1d22b99ede51 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.2.0_1608159566596/spec.json @@ -0,0 +1,178 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159566596, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`D1` STRING KEY, `D1` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `D0` INTEGER, `D1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`D1` STRING KEY, `D1` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`D1` STRING KEY, `KSQL_COL_0` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "UDAF nested in UDF in select expression (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : "0,x" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,x" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,xxx" + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "3,xxx" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,yy" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "x", + "value" : "M" + }, { + "topic" : "OUTPUT", + "key" : "x", + "value" : "r" + }, { + "topic" : "OUTPUT", + "key" : "xxx", + "value" : "M" + }, { + "topic" : "OUTPUT", + "key" : "xxx", + "value" : "r" + }, { + "topic" : "OUTPUT", + "key" : "x", + "value" : "M" + }, { + "topic" : "OUTPUT", + "key" : "xxx", + "value" : "M" + }, { + "topic" : "OUTPUT", + "key" : "yy", + "value" : "M" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, d0 INT, d1 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT d1, SUBSTRING('Mr Bugalicious', CAST(COUNT(*) AS INT), 1) FROM TEST GROUP BY d1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`D1` STRING KEY, `KSQL_COL_0` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `D0` INTEGER, `D1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.2.0_1608159566596/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.2.0_1608159566596/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDAF_nested_in_UDF_in_select_expression_(table-_table)/6.2.0_1608159566596/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDF_nested_in_UDAF_in_select_expression_(stream-_table)/6.2.0_1608159566734/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDF_nested_in_UDAF_in_select_expression_(stream-_table)/6.2.0_1608159566734/plan.json new file mode 100644 index 000000000000..747cecbce739 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDF_nested_in_UDAF_in_select_expression_(stream-_table)/6.2.0_1608159566734/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, D1 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `D1` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.D1 D1,\n SUM(LEN(TEST.D1)) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.D1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`D1` STRING KEY, `KSQL_COL_0` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `D1` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "D1 AS D1", "LEN(D1) AS KSQL_INTERNAL_COL_1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "D1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "D1" ], + "aggregationFunctions" : [ "SUM(KSQL_INTERNAL_COL_1)" ] + }, + "keyColumnNames" : [ "D1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDF_nested_in_UDAF_in_select_expression_(stream-_table)/6.2.0_1608159566734/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDF_nested_in_UDAF_in_select_expression_(stream-_table)/6.2.0_1608159566734/spec.json new file mode 100644 index 000000000000..d8d1de95ad10 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDF_nested_in_UDAF_in_select_expression_(stream-_table)/6.2.0_1608159566734/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159566734, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`D1` STRING KEY, `D1` STRING, `KSQL_INTERNAL_COL_1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `D1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`D1` STRING KEY, `D1` STRING, `KSQL_AGG_VARIABLE_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`D1` STRING KEY, `KSQL_COL_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "UDF nested in UDAF in select expression (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "x" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "xxx" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "y" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "x" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "xxx" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "x", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "xxx", + "value" : "3" + }, { + "topic" : "OUTPUT", + "key" : "y", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "x", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "xxx", + "value" : "6" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, d1 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT d1, SUM(LEN(d1)) FROM TEST GROUP BY d1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`D1` STRING KEY, `KSQL_COL_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `D1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDF_nested_in_UDAF_in_select_expression_(stream-_table)/6.2.0_1608159566734/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDF_nested_in_UDAF_in_select_expression_(stream-_table)/6.2.0_1608159566734/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_UDF_nested_in_UDAF_in_select_expression_(stream-_table)/6.2.0_1608159566734/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(stream-_table)/6.2.0_1608159565255/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(stream-_table)/6.2.0_1608159565255/plan.json new file mode 100644 index 000000000000..9e664b5b1d1e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(stream-_table)/6.2.0_1608159565255/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, F1 INTEGER, F2 INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n (TEST.F2 - TEST.F1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY (TEST.F2 - TEST.F1)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F1` INTEGER, `F2` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F2 AS F2", "F1 AS F1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "(F2 - F1)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F2", "F1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(stream-_table)/6.2.0_1608159565255/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(stream-_table)/6.2.0_1608159565255/spec.json new file mode 100644 index 000000000000..4e3c8353284c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(stream-_table)/6.2.0_1608159565255/spec.json @@ -0,0 +1,150 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159565255, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `F2` INTEGER, `F1` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `F2` INTEGER, `F1` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "arithmetic binary expression with projection in-order & non-commutative group by (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "1,2" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "2,3" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "2,4" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "6,8" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f2 - f1, COUNT(*) FROM TEST GROUP BY f2 - f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(stream-_table)/6.2.0_1608159565255/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(stream-_table)/6.2.0_1608159565255/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(stream-_table)/6.2.0_1608159565255/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.2.0_1608159565350/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.2.0_1608159565350/plan.json new file mode 100644 index 000000000000..a5a3d24136be --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.2.0_1608159565350/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, F0 INTEGER, F1 INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n (TEST.F0 - TEST.F1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY (TEST.F0 - TEST.F1)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F0 AS F0", "F1 AS F1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "(F0 - F1)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F0", "F1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.2.0_1608159565350/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.2.0_1608159565350/spec.json new file mode 100644 index 000000000000..b29c89dbfea3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.2.0_1608159565350/spec.json @@ -0,0 +1,170 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159565350, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `F0` INTEGER, `F1` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `F0` INTEGER, `F1` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "arithmetic binary expression with projection in-order & non-commutative group by (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,0" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,1" + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "3,1" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "4,2" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, f0 INT, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f0 - f1, COUNT(*) FROM TEST GROUP BY f0 - f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.2.0_1608159565350/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.2.0_1608159565350/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_binary_expression_with_projection_in-order_&_non-commutative_group_by_(table-_table)/6.2.0_1608159565350/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_in_group_by_column_used_in_non-aggregate_function_in_select/6.2.0_1608159558896/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_in_group_by_column_used_in_non-aggregate_function_in_select/6.2.0_1608159558896/plan.json new file mode 100644 index 000000000000..ba62cd6cf653 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_in_group_by_column_used_in_non-aggregate_function_in_select/6.2.0_1608159558896/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID INTEGER KEY, COL1 INTEGER, COL2 INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `COL1` INTEGER, `COL2` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n (INPUT.COL1 + INPUT.COL2) G1,\n AS_VALUE((INPUT.COL1 + INPUT.COL2)) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM INPUT INPUT\nGROUP BY (INPUT.COL1 + INPUT.COL2)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`G1` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `COL1` INTEGER, `COL2` INTEGER" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "COL1 AS COL1", "COL2 AS COL2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "(COL1 + COL2)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "COL1", "COL2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "G1" ], + "selectExpressions" : [ "AS_VALUE((COL1 + COL2)) AS KSQL_COL_0", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_in_group_by_column_used_in_non-aggregate_function_in_select/6.2.0_1608159558896/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_in_group_by_column_used_in_non-aggregate_function_in_select/6.2.0_1608159558896/spec.json new file mode 100644 index 000000000000..1b40d2b257a2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_in_group_by_column_used_in_non-aggregate_function_in_select/6.2.0_1608159558896/spec.json @@ -0,0 +1,174 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159558896, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `COL1` INTEGER, `COL2` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `COL1` INTEGER, `COL2` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `COL1` INTEGER, `COL2` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`G1` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "arithmetic in group by column used in non-aggregate function in select", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "col1" : 1, + "col2" : 1 + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "col1" : 2, + "col2" : 2 + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "col1" : 3, + "col2" : 3 + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "col1" : 4, + "col2" : 4 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "KSQL_COL_0" : 2, + "KSQL_COL_1" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 4, + "value" : { + "KSQL_COL_0" : 4, + "KSQL_COL_1" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 6, + "value" : { + "KSQL_COL_0" : 6, + "KSQL_COL_1" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 8, + "value" : { + "KSQL_COL_0" : 8, + "KSQL_COL_1" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (id INT KEY, col1 INT, col2 INT) WITH (kafka_topic='test_topic', value_format='json');", "CREATE TABLE OUTPUT AS SELECT col1+col2 AS G1, AS_VALUE(col1+col2), COUNT(*) FROM input GROUP BY col1+col2;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `COL1` INTEGER, `COL2` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`G1` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_in_group_by_column_used_in_non-aggregate_function_in_select/6.2.0_1608159558896/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_in_group_by_column_used_in_non-aggregate_function_in_select/6.2.0_1608159558896/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_arithmetic_in_group_by_column_used_in_non-aggregate_function_in_select/6.2.0_1608159558896/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_by_non-STRING_key/6.2.0_1608159567188/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_by_non-STRING_key/6.2.0_1608159567188/plan.json new file mode 100644 index 000000000000..8d2e6ea9a708 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_by_non-STRING_key/6.2.0_1608159567188/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K STRING KEY, F0 INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` STRING KEY, `F0` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.F0 F0,\n COUNT(1) KSQL_COL_0\nFROM INPUT INPUT\nGROUP BY INPUT.F0\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F0` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F0` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F0 AS F0", "1 AS KSQL_INTERNAL_COL_1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F0" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F0" ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_1)" ] + }, + "keyColumnNames" : [ "F0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_by_non-STRING_key/6.2.0_1608159567188/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_by_non-STRING_key/6.2.0_1608159567188/spec.json new file mode 100644 index 000000000000..8d8c49160e0c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_by_non-STRING_key/6.2.0_1608159567188/spec.json @@ -0,0 +1,142 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159567188, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F0` INTEGER KEY, `F0` INTEGER, `KSQL_INTERNAL_COL_1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `F0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F0` INTEGER KEY, `F0` INTEGER, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F0` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "by non-STRING key", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "2" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "3" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "2" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 2, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K STRING KEY, f0 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f0, COUNT(1) FROM INPUT GROUP BY f0;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `F0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F0` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_by_non-STRING_key/6.2.0_1608159567188/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_by_non-STRING_key/6.2.0_1608159567188/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_by_non-STRING_key/6.2.0_1608159567188/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params/6.2.0_1608159559663/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params/6.2.0_1608159559663/plan.json new file mode 100644 index 000000000000..0faaeacbac18 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params/6.2.0_1608159559663/plan.json @@ -0,0 +1,183 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (V0 INTEGER KEY, V1 INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`V0` INTEGER KEY, `V1` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.V0 V0,\n TEST.V1 V1,\n SUM((TEST.V0 + TEST.V1)) SUM\nFROM TEST TEST\nGROUP BY TEST.V0, TEST.V1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`V0` INTEGER KEY, `V1` INTEGER KEY, `SUM` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`V0` INTEGER KEY, `V1` INTEGER" + }, + "keyColumnNames" : [ "V0" ], + "selectExpressions" : [ "V0 AS V0", "V1 AS V1", "(V0 + V1) AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "V0", "V1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "V0", "V1" ], + "aggregationFunctions" : [ "SUM(KSQL_INTERNAL_COL_2)" ] + }, + "keyColumnNames" : [ "V0", "V1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS SUM" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params/6.2.0_1608159559663/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params/6.2.0_1608159559663/spec.json new file mode 100644 index 000000000000..caaece3fda00 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params/6.2.0_1608159559663/spec.json @@ -0,0 +1,165 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159559663, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`V0` INTEGER KEY, `V1` INTEGER KEY, `V0` INTEGER, `V1` INTEGER, `KSQL_INTERNAL_COL_2` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`V0` INTEGER KEY, `V1` INTEGER", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`V0` INTEGER KEY, `V1` INTEGER KEY, `V0` INTEGER, `V1` INTEGER, `KSQL_AGG_VARIABLE_0` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`V0` INTEGER KEY, `V1` INTEGER KEY, `SUM` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "complex UDAF params", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "V1" : 10 + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "V1" : 20 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "V1" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "V0" : 0, + "V1" : 10 + }, + "value" : { + "SUM" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : { + "V0" : 1, + "V1" : 20 + }, + "value" : { + "SUM" : 21 + } + }, { + "topic" : "OUTPUT", + "key" : { + "V0" : 0, + "V1" : 10 + }, + "value" : { + "SUM" : 20 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (V0 INT KEY, V1 INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT V0, V1, SUM(V0 + V1) AS SUM FROM TEST GROUP BY V0, V1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`V0` INTEGER KEY, `V1` INTEGER KEY, `SUM` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`V0` INTEGER KEY, `V1` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params/6.2.0_1608159559663/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params/6.2.0_1608159559663/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params/6.2.0_1608159559663/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_GROUP_BY/6.2.0_1608159559757/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_GROUP_BY/6.2.0_1608159559757/plan.json new file mode 100644 index 000000000000..e0b70d821816 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_GROUP_BY/6.2.0_1608159559757/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (V0 INTEGER KEY, V1 INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`V0` INTEGER KEY, `V1` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n (TEST.V0 + TEST.V1) NEW_KEY,\n SUM((TEST.V0 + TEST.V1)) SUM\nFROM TEST TEST\nGROUP BY (TEST.V0 + TEST.V1)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NEW_KEY` INTEGER KEY, `SUM` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`V0` INTEGER KEY, `V1` INTEGER" + }, + "keyColumnNames" : [ "V0" ], + "selectExpressions" : [ "V0 AS V0", "V1 AS V1", "(V0 + V1) AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "(V0 + V1)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "V0", "V1" ], + "aggregationFunctions" : [ "SUM(KSQL_INTERNAL_COL_2)" ] + }, + "keyColumnNames" : [ "NEW_KEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS SUM" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_GROUP_BY/6.2.0_1608159559757/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_GROUP_BY/6.2.0_1608159559757/spec.json new file mode 100644 index 000000000000..79a52b1fe8ca --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_GROUP_BY/6.2.0_1608159559757/spec.json @@ -0,0 +1,154 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159559757, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `V0` INTEGER, `V1` INTEGER, `KSQL_INTERNAL_COL_2` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`V0` INTEGER KEY, `V1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `V0` INTEGER, `V1` INTEGER, `KSQL_AGG_VARIABLE_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NEW_KEY` INTEGER KEY, `SUM` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "complex UDAF params matching GROUP BY", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "V1" : 10 + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "V1" : 20 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "V1" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "SUM" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : 21, + "value" : { + "SUM" : 21 + } + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "SUM" : 20 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (V0 INT KEY, V1 INT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT (V0 + V1) AS NEW_KEY, SUM(V0 + V1) AS SUM FROM TEST GROUP BY V0 + V1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NEW_KEY` INTEGER KEY, `SUM` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`V0` INTEGER KEY, `V1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_GROUP_BY/6.2.0_1608159559757/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_GROUP_BY/6.2.0_1608159559757/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_GROUP_BY/6.2.0_1608159559757/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_HAVING/6.2.0_1608159559837/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_HAVING/6.2.0_1608159559837/plan.json new file mode 100644 index 000000000000..3a6bc0086017 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_HAVING/6.2.0_1608159559837/plan.json @@ -0,0 +1,190 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (V0 INTEGER KEY, V1 INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`V0` INTEGER KEY, `V1` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n SUM((TEST.V0 + TEST.V1)) SUM,\n TEST.V0 V0,\n TEST.V1 V1\nFROM TEST TEST\nGROUP BY TEST.V0, TEST.V1\nHAVING ((TEST.V0 + TEST.V1) <= 20)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`V0` INTEGER KEY, `V1` INTEGER KEY, `SUM` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`V0` INTEGER KEY, `V1` INTEGER" + }, + "keyColumnNames" : [ "V0" ], + "selectExpressions" : [ "V0 AS V0", "V1 AS V1", "(V0 + V1) AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "V0", "V1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "V0", "V1" ], + "aggregationFunctions" : [ "SUM(KSQL_INTERNAL_COL_2)" ] + }, + "filterExpression" : "((V0 + V1) <= 20)" + }, + "keyColumnNames" : [ "V0", "V1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS SUM" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_HAVING/6.2.0_1608159559837/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_HAVING/6.2.0_1608159559837/spec.json new file mode 100644 index 000000000000..006643c49e93 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_HAVING/6.2.0_1608159559837/spec.json @@ -0,0 +1,156 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159559837, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`V0` INTEGER KEY, `V1` INTEGER KEY, `V0` INTEGER, `V1` INTEGER, `KSQL_INTERNAL_COL_2` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`V0` INTEGER KEY, `V1` INTEGER", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`V0` INTEGER KEY, `V1` INTEGER KEY, `V0` INTEGER, `V1` INTEGER, `KSQL_AGG_VARIABLE_0` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`V0` INTEGER KEY, `V1` INTEGER KEY, `SUM` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "complex UDAF params matching HAVING", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "V1" : 10 + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "V1" : 20 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "V1" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "V0" : 0, + "V1" : 10 + }, + "value" : { + "SUM" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : { + "V0" : 0, + "V1" : 10 + }, + "value" : { + "SUM" : 20 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (V0 INT KEY, V1 INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT SUM(V0 + V1) AS SUM, V0, V1 FROM TEST GROUP BY V0, V1 HAVING V0 + V1 <= 20;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`V0` INTEGER KEY, `V1` INTEGER KEY, `SUM` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`V0` INTEGER KEY, `V1` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_HAVING/6.2.0_1608159559837/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_HAVING/6.2.0_1608159559837/topology new file mode 100644 index 000000000000..b08ea0de8518 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_complex_UDAF_params_matching_HAVING/6.2.0_1608159559837/topology @@ -0,0 +1,49 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000014 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000014 (stores: []) + --> KSTREAM-SINK-0000000015 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000015 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000014 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(stream-_table)/6.2.0_1608159564472/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(stream-_table)/6.2.0_1608159564472/plan.json new file mode 100644 index 000000000000..350b957d94e7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(stream-_table)/6.2.0_1608159564472/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, IGNORED STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `IGNORED` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n 1 KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY 1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `IGNORED` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(stream-_table)/6.2.0_1608159564472/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(stream-_table)/6.2.0_1608159564472/spec.json new file mode 100644 index 000000000000..b067a65c15ab --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(stream-_table)/6.2.0_1608159564472/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159564472, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `IGNORED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "constant (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "-" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "-" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "-" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "-" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "-" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "3" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "4" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "5" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, ignored VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT 1, COUNT(*) FROM TEST GROUP BY 1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `IGNORED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(stream-_table)/6.2.0_1608159564472/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(stream-_table)/6.2.0_1608159564472/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(stream-_table)/6.2.0_1608159564472/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.2.0_1608159564583/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.2.0_1608159564583/plan.json new file mode 100644 index 000000000000..5b0a7d6a14e6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.2.0_1608159564583/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K INTEGER PRIMARY KEY, USER INTEGER, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n 1 KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY 1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.2.0_1608159564583/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.2.0_1608159564583/spec.json new file mode 100644 index 000000000000..1ea990727a4c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.2.0_1608159564583/spec.json @@ -0,0 +1,170 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159564583, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "constant (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,r0" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r1" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3,r0" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : null + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r0" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "3" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT 1, COUNT(*) FROM TEST GROUP BY 1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.2.0_1608159564583/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.2.0_1608159564583/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_constant_(table-_table)/6.2.0_1608159564583/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_double_field_with_re-key_(stream-_table)/6.2.0_1608159562361/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_double_field_with_re-key_(stream-_table)/6.2.0_1608159562361/plan.json new file mode 100644 index 000000000000..0ba7267685b9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_double_field_with_re-key_(stream-_table)/6.2.0_1608159562361/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, DATA DOUBLE) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `DATA` DOUBLE", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA DATA,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DATA` DOUBLE KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `DATA` DOUBLE" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "DATA" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_double_field_with_re-key_(stream-_table)/6.2.0_1608159562361/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_double_field_with_re-key_(stream-_table)/6.2.0_1608159562361/spec.json new file mode 100644 index 000000000000..b99b8994e76d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_double_field_with_re-key_(stream-_table)/6.2.0_1608159562361/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159562361, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` DOUBLE KEY, `DATA` DOUBLE, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `DATA` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` DOUBLE KEY, `DATA` DOUBLE, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DATA` DOUBLE KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "double field with re-key (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "0.1" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "0.2" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "0.1" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "0.2" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "0.1" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0.1, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 0.2, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 0.1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 0.2, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 0.1, + "value" : "3" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, data double) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) FROM TEST GROUP BY DATA;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DATA` DOUBLE KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `DATA` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_double_field_with_re-key_(stream-_table)/6.2.0_1608159562361/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_double_field_with_re-key_(stream-_table)/6.2.0_1608159562361/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_double_field_with_re-key_(stream-_table)/6.2.0_1608159562361/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_fields_(stream-_table)/6.2.0_1608159566314/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_fields_(stream-_table)/6.2.0_1608159566314/plan.json new file mode 100644 index 000000000000..a9a0ab36457c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_fields_(stream-_table)/6.2.0_1608159566314/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA DATA,\n COUNT(1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1,\n AS_VALUE(TEST.DATA) COPY\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT, `KSQL_COL_1` BIGINT, `COPY` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME", "1 AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_2)", "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "DATA" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0", "KSQL_AGG_VARIABLE_1 AS KSQL_COL_1", "AS_VALUE(DATA) AS COPY" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_fields_(stream-_table)/6.2.0_1608159566314/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_fields_(stream-_table)/6.2.0_1608159566314/spec.json new file mode 100644 index 000000000000..2c3c175aff3a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_fields_(stream-_table)/6.2.0_1608159566314/spec.json @@ -0,0 +1,166 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159566314, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_INTERNAL_COL_2` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT, `KSQL_AGG_VARIABLE_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT, `KSQL_COL_1` BIGINT, `COPY` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "duplicate fields (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d2" + }, + "timestamp" : 2 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + }, + "timestamp" : 3 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 1, + "KSQL_COL_1" : 1, + "COPY" : "d1" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_0" : 1, + "KSQL_COL_1" : 1, + "COPY" : "d2" + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 2, + "KSQL_COL_1" : 2, + "COPY" : "d1" + }, + "timestamp" : 3 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(1), COUNT(*), AS_VALUE(DATA) AS COPY FROM TEST GROUP BY data;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT, `KSQL_COL_1` BIGINT, `COPY` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_fields_(stream-_table)/6.2.0_1608159566314/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_fields_(stream-_table)/6.2.0_1608159566314/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_fields_(stream-_table)/6.2.0_1608159566314/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_udafs_(stream-_table)/6.2.0_1608159566420/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_udafs_(stream-_table)/6.2.0_1608159566420/plan.json new file mode 100644 index 000000000000..995466d08f5a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_udafs_(stream-_table)/6.2.0_1608159566420/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA DATA,\n COUNT(1) KSQL_COL_0,\n COUNT(1) KSQL_COL_1\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "DATA AS DATA", "1 AS KSQL_INTERNAL_COL_1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA" ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_1)", "COUNT(KSQL_INTERNAL_COL_1)" ] + }, + "keyColumnNames" : [ "DATA" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0", "KSQL_AGG_VARIABLE_1 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_udafs_(stream-_table)/6.2.0_1608159566420/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_udafs_(stream-_table)/6.2.0_1608159566420/spec.json new file mode 100644 index 000000000000..972ade059fa7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_udafs_(stream-_table)/6.2.0_1608159566420/spec.json @@ -0,0 +1,163 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159566420, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `KSQL_INTERNAL_COL_1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `KSQL_AGG_VARIABLE_0` BIGINT, `KSQL_AGG_VARIABLE_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "duplicate udafs (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d2" + }, + "timestamp" : 2 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + }, + "timestamp" : 3 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 1, + "KSQL_COL_1" : 1 + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_0" : 1, + "KSQL_COL_1" : 1 + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 2, + "KSQL_COL_1" : 2 + }, + "timestamp" : 3 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(1), COUNT(1) FROM TEST GROUP BY data;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_udafs_(stream-_table)/6.2.0_1608159566420/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_udafs_(stream-_table)/6.2.0_1608159566420/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_duplicate_udafs_(stream-_table)/6.2.0_1608159566420/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_expressions_used_in_non-aggregate_function_in_select_whose_children_are_not_part_of_group-by/6.2.0_1608159559005/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_expressions_used_in_non-aggregate_function_in_select_whose_children_are_not_part_of_group-by/6.2.0_1608159559005/plan.json new file mode 100644 index 000000000000..7b7c4eb5dfdd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_expressions_used_in_non-aggregate_function_in_select_whose_children_are_not_part_of_group-by/6.2.0_1608159559005/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID INTEGER KEY, COL1 MAP, COL2 MAP) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `COL1` MAP, `COL2` MAP", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n (INPUT.COL1['foo'] + INPUT.COL2['bar']) G1,\n AS_VALUE((INPUT.COL1['foo'] + INPUT.COL2['bar'])) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM INPUT INPUT\nGROUP BY (INPUT.COL1['foo'] + INPUT.COL2['bar'])\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`G1` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `COL1` MAP, `COL2` MAP" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "COL1 AS COL1", "COL2 AS COL2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "(COL1['foo'] + COL2['bar'])" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "COL1", "COL2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "G1" ], + "selectExpressions" : [ "AS_VALUE((COL1['foo'] + COL2['bar'])) AS KSQL_COL_0", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_expressions_used_in_non-aggregate_function_in_select_whose_children_are_not_part_of_group-by/6.2.0_1608159559005/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_expressions_used_in_non-aggregate_function_in_select_whose_children_are_not_part_of_group-by/6.2.0_1608159559005/spec.json new file mode 100644 index 000000000000..b5045620797c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_expressions_used_in_non-aggregate_function_in_select_whose_children_are_not_part_of_group-by/6.2.0_1608159559005/spec.json @@ -0,0 +1,169 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159559005, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `COL1` MAP, `COL2` MAP, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `COL1` MAP, `COL2` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `COL1` MAP, `COL2` MAP, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`G1` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "expressions used in non-aggregate function in select whose children are not part of group-by", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "col1" : { + "a" : 1 + }, + "col2" : { + "b" : 1 + } + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "col1" : { + "foo" : 1 + }, + "col2" : { + "bar" : 1 + } + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "col1" : { + "bar" : 1 + }, + "col2" : { + "foo" : 1 + } + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "col1" : { + "foo" : 1 + }, + "col2" : { + "foo" : 1 + } + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "KSQL_COL_0" : 2, + "KSQL_COL_1" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (id INT KEY, col1 MAP, col2 MAP) WITH (kafka_topic='test_topic', value_format='json');", "CREATE TABLE OUTPUT AS SELECT col1['foo']+col2['bar'] AS G1, AS_VALUE(col1['foo']+col2['bar']), COUNT(*) FROM input GROUP BY col1['foo']+col2['bar'];" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `COL1` MAP, `COL2` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`G1` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_expressions_used_in_non-aggregate_function_in_select_whose_children_are_not_part_of_group-by/6.2.0_1608159559005/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_expressions_used_in_non-aggregate_function_in_select_whose_children_are_not_part_of_group-by/6.2.0_1608159559005/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_expressions_used_in_non-aggregate_function_in_select_whose_children_are_not_part_of_group-by/6.2.0_1608159559005/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(stream-_table)/6.2.0_1608159564716/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(stream-_table)/6.2.0_1608159564716/plan.json new file mode 100644 index 000000000000..e0285c7e743c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(stream-_table)/6.2.0_1608159564716/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, F1 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F1` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n SUBSTRING(TEST.F1, 0, 1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F1` STRING KEY, `KSQL_COL_0` STRING, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F1` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F1 AS F1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F1" ], + "selectExpressions" : [ "SUBSTRING(F1, 0, 1) AS KSQL_COL_0", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(stream-_table)/6.2.0_1608159564716/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(stream-_table)/6.2.0_1608159564716/spec.json new file mode 100644 index 000000000000..bc98350d3228 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(stream-_table)/6.2.0_1608159564716/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159564716, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F1` STRING KEY, `F1` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F1` STRING KEY, `F1` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F1` STRING KEY, `KSQL_COL_0` STRING, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "field with field used in function in projection (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "one" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "two" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "three" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "one" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "five" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "one", + "value" : "o,1" + }, { + "topic" : "OUTPUT", + "key" : "two", + "value" : "t,1" + }, { + "topic" : "OUTPUT", + "key" : "three", + "value" : "t,1" + }, { + "topic" : "OUTPUT", + "key" : "one", + "value" : "o,2" + }, { + "topic" : "OUTPUT", + "key" : "five", + "value" : "f,1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, f1 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, SUBSTRING(f1, 0, 1), COUNT(*) FROM TEST GROUP BY f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F1` STRING KEY, `KSQL_COL_0` STRING, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(stream-_table)/6.2.0_1608159564716/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(stream-_table)/6.2.0_1608159564716/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(stream-_table)/6.2.0_1608159564716/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.2.0_1608159564815/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.2.0_1608159564815/plan.json new file mode 100644 index 000000000000..5cf2d18b4fdf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.2.0_1608159564815/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, USER INTEGER, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.REGION REGION,\n SUBSTRING(TEST.REGION, 2, 1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` STRING, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "REGION AS REGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "SUBSTRING(REGION, 2, 1) AS KSQL_COL_0", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.2.0_1608159564815/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.2.0_1608159564815/spec.json new file mode 100644 index 000000000000..6c1809f97b9c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.2.0_1608159564815/spec.json @@ -0,0 +1,170 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159564815, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` STRING, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "field with field used in function in projection (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,r0" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r1" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3,r0" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : null + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r0" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "0,1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "1,1" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "0,2" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "0,1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "1,0" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "0,2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT region, SUBSTRING(region, 2, 1), COUNT(*) FROM TEST GROUP BY region;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` STRING, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.2.0_1608159564815/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.2.0_1608159564815/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_field_used_in_function_in_projection_(table-_table)/6.2.0_1608159564815/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)/6.2.0_1608159562252/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)/6.2.0_1608159562252/plan.json new file mode 100644 index 000000000000..a93ff991106e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)/6.2.0_1608159562252/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA DATA,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "DATA" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)/6.2.0_1608159562252/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)/6.2.0_1608159562252/spec.json new file mode 100644 index 000000000000..82887664a7bf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)/6.2.0_1608159562252/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159562252, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "field with re-key (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "d1" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "d2" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "d1" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "d2" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "d1" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : "3" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) FROM TEST GROUP BY DATA;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)/6.2.0_1608159562252/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)/6.2.0_1608159562252/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)/6.2.0_1608159562252/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_AVRO/6.2.0_1608159562480/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_AVRO/6.2.0_1608159562480/plan.json new file mode 100644 index 000000000000..a1e6e244abb6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_AVRO/6.2.0_1608159562480/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA DATA,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`K` STRING KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "DATA" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_AVRO/6.2.0_1608159562480/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_AVRO/6.2.0_1608159562480/spec.json new file mode 100644 index 000000000000..707e4c055c27 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_AVRO/6.2.0_1608159562480/spec.json @@ -0,0 +1,243 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159562480, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "field with re-key (stream->table) - format - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d2" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d2" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "DATA", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) FROM TEST GROUP BY DATA;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "DATA", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "DATA", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "DATA", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "KSQL_COL_0", + "type" : [ "null", "long" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_AVRO/6.2.0_1608159562480/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_AVRO/6.2.0_1608159562480/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_AVRO/6.2.0_1608159562480/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_JSON/6.2.0_1608159562600/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_JSON/6.2.0_1608159562600/plan.json new file mode 100644 index 000000000000..5df01b505cb7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_JSON/6.2.0_1608159562600/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA DATA,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "DATA" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_JSON/6.2.0_1608159562600/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_JSON/6.2.0_1608159562600/spec.json new file mode 100644 index 000000000000..9d4ba365cbc8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_JSON/6.2.0_1608159562600/spec.json @@ -0,0 +1,178 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159562600, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "field with re-key (stream->table) - format - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d2" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d2" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) FROM TEST GROUP BY DATA;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_JSON/6.2.0_1608159562600/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_JSON/6.2.0_1608159562600/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_JSON/6.2.0_1608159562600/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159562737/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159562737/plan.json new file mode 100644 index 000000000000..3049961718ff --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159562737/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA DATA,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`K` STRING KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "DATA" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159562737/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159562737/spec.json new file mode 100644 index 000000000000..7da27f79f395 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159562737/spec.json @@ -0,0 +1,184 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159562737, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + } + }, + "testCase" : { + "name" : "field with re-key (stream->table) - format - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d2" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d2" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string DATA = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) FROM TEST GROUP BY DATA;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string DATA = 1;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string DATA = 1;\n int64 ROWTIME = 2;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string DATA = 1;\n int64 ROWTIME = 2;\n int64 KSQL_AGG_VARIABLE_0 = 3;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 KSQL_COL_0 = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159562737/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159562737/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159562737/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.2.0_1608159562861/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.2.0_1608159562861/plan.json new file mode 100644 index 000000000000..1b08117f443c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.2.0_1608159562861/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, USER INTEGER, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.REGION REGION,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "REGION AS REGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.2.0_1608159562861/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.2.0_1608159562861/spec.json new file mode 100644 index 000000000000..a3bbfa13c985 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.2.0_1608159562861/spec.json @@ -0,0 +1,170 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159562861, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "field with re-key (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,r0" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r1" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3,r0" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : null + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r0" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT region, COUNT(*) FROM TEST GROUP BY region;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.2.0_1608159562861/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.2.0_1608159562861/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_field_with_re-key_(table-_table)/6.2.0_1608159562861/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)/6.2.0_1608159560635/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)/6.2.0_1608159560635/plan.json new file mode 100644 index 000000000000..70c71f34391c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)/6.2.0_1608159560635/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, F1 INTEGER, F2 STRING) WITH (FORMAT='DELIMITED', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F2", "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)/6.2.0_1608159560635/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)/6.2.0_1608159560635/spec.json new file mode 100644 index 000000000000..9e8863398caa --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)/6.2.0_1608159560635/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159560635, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "fields (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "3,a" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "a,1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "b,2", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "a,1", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "b,2", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "a,3", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)/6.2.0_1608159560635/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)/6.2.0_1608159560635/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)/6.2.0_1608159560635/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_AVRO/6.2.0_1608159560861/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_AVRO/6.2.0_1608159560861/plan.json new file mode 100644 index 000000000000..f67f1b119f20 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_AVRO/6.2.0_1608159560861/plan.json @@ -0,0 +1,183 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='JSON', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F2", "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_AVRO/6.2.0_1608159560861/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_AVRO/6.2.0_1608159560861/spec.json new file mode 100644 index 000000000000..e1e061670886 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_AVRO/6.2.0_1608159560861/spec.json @@ -0,0 +1,281 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159560861, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "fields (stream->table) - format - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "F1" : 3, + "F2" : "a" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 3 + }, + "value" : { + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', key_format='JSON', value_format='AVRO');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "KSQL_COL_0", + "type" : [ "null", "long" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_AVRO/6.2.0_1608159560861/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_AVRO/6.2.0_1608159560861/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_AVRO/6.2.0_1608159560861/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_JSON/6.2.0_1608159560994/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_JSON/6.2.0_1608159560994/plan.json new file mode 100644 index 000000000000..eddc7c0558e4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_JSON/6.2.0_1608159560994/plan.json @@ -0,0 +1,183 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='JSON', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F2", "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_JSON/6.2.0_1608159560994/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_JSON/6.2.0_1608159560994/spec.json new file mode 100644 index 000000000000..db7821ec5d74 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_JSON/6.2.0_1608159560994/spec.json @@ -0,0 +1,200 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159560994, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "fields (stream->table) - format - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "F1" : 3, + "F2" : "a" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 3 + }, + "value" : { + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', key_format='JSON', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_JSON/6.2.0_1608159560994/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_JSON/6.2.0_1608159560994/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_JSON/6.2.0_1608159560994/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159561142/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159561142/plan.json new file mode 100644 index 000000000000..154662d7d36e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159561142/plan.json @@ -0,0 +1,183 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='JSON', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F2", "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159561142/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159561142/spec.json new file mode 100644 index 000000000000..5d03bd87bcc8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159561142/spec.json @@ -0,0 +1,206 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159561142, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + } + }, + "testCase" : { + "name" : "fields (stream->table) - format - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "F1" : 3, + "F2" : "a" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 3 + }, + "value" : { + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 F1 = 1;\n string F2 = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', key_format='JSON', value_format='PROTOBUF');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 F1 = 1;\n string F2 = 2;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 F1 = 1;\n string F2 = 2;\n int64 ROWTIME = 3;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 F1 = 1;\n string F2 = 2;\n int64 ROWTIME = 3;\n int64 KSQL_AGG_VARIABLE_0 = 4;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 KSQL_COL_0 = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159561142/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159561142/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(stream-_table)_-_format_-_PROTOBUF/6.2.0_1608159561142/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.2.0_1608159561395/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.2.0_1608159561395/plan.json new file mode 100644 index 000000000000..3c5b05ec0c25 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.2.0_1608159561395/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (FORMAT='DELIMITED', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F2", "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.2.0_1608159561395/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.2.0_1608159561395/spec.json new file mode 100644 index 000000000000..4f6b75b88614 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.2.0_1608159561395/spec.json @@ -0,0 +1,174 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159561395, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "fields (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,b" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : null + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "a,1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "b,2", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "a,1", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "b,1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "b,2", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "b,1", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "a,1", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.2.0_1608159561395/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.2.0_1608159561395/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)/6.2.0_1608159561395/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.2.0_1608159561751/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.2.0_1608159561751/plan.json new file mode 100644 index 000000000000..c14ab8627cec --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.2.0_1608159561751/plan.json @@ -0,0 +1,184 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='JSON', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F2", "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.2.0_1608159561751/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.2.0_1608159561751/spec.json new file mode 100644 index 000000000000..e15f144b1f3d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.2.0_1608159561751/spec.json @@ -0,0 +1,319 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159561751, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "fields (table->table) - format - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : null + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', key_format='JSON', value_format='AVRO');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "KSQL_COL_0", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "F1", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "F2", + "type" : [ "null", "string" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.2.0_1608159561751/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.2.0_1608159561751/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_AVRO/6.2.0_1608159561751/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.2.0_1608159561944/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.2.0_1608159561944/plan.json new file mode 100644 index 000000000000..8df72363c413 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.2.0_1608159561944/plan.json @@ -0,0 +1,184 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='JSON', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F2", "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.2.0_1608159561944/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.2.0_1608159561944/spec.json new file mode 100644 index 000000000000..54c628d8f54f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.2.0_1608159561944/spec.json @@ -0,0 +1,224 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159561944, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "fields (table->table) - format - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : null + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', key_format='JSON', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.2.0_1608159561944/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.2.0_1608159561944/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_JSON/6.2.0_1608159561944/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.2.0_1608159562098/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.2.0_1608159562098/plan.json new file mode 100644 index 000000000000..8b94a6cdb711 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.2.0_1608159562098/plan.json @@ -0,0 +1,184 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='JSON', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F2", "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.2.0_1608159562098/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.2.0_1608159562098/spec.json new file mode 100644 index 000000000000..1604eca69dd3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.2.0_1608159562098/spec.json @@ -0,0 +1,231 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159562098, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + } + }, + "testCase" : { + "name" : "fields (table->table) - format - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "F1" : 2, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "b" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : null + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "F1" : 1, + "F2" : "a" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 2 + }, + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "b", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F2" : "a", + "F1" : 1 + }, + "value" : { + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 F1 = 1;\n string F2 = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', key_format='JSON', value_format='PROTOBUF');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 F1 = 1;\n string F2 = 2;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 F1 = 1;\n string F2 = 2;\n int64 ROWTIME = 3;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 F1 = 1;\n string F2 = 2;\n int64 ROWTIME = 3;\n int64 KSQL_AGG_VARIABLE_0 = 4;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 KSQL_COL_0 = 1;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 F1 = 1;\n string F2 = 2;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.2.0_1608159562098/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.2.0_1608159562098/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_(table-_table)_-_format_-_PROTOBUF/6.2.0_1608159562098/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.2.0_1608159561540/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.2.0_1608159561540/plan.json new file mode 100644 index 000000000000..9457b024affe --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.2.0_1608159561540/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (FORMAT='DELIMITED', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n AS_VALUE(TEST.F1) F3,\n AS_VALUE(TEST.F2) F4,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F3` INTEGER, `F4` STRING, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F2", "F1" ], + "selectExpressions" : [ "AS_VALUE(F1) AS F3", "AS_VALUE(F2) AS F4", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.2.0_1608159561540/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.2.0_1608159561540/spec.json new file mode 100644 index 000000000000..7d745b355ecb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.2.0_1608159561540/spec.json @@ -0,0 +1,174 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159561540, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F3` INTEGER, `F4` STRING, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "fields - copied into value (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,b" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : null + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "a,1", + "value" : "1,a,1" + }, { + "topic" : "OUTPUT", + "key" : "b,2", + "value" : "2,b,1" + }, { + "topic" : "OUTPUT", + "key" : "a,1", + "value" : "1,a,0" + }, { + "topic" : "OUTPUT", + "key" : "b,1", + "value" : "1,b,1" + }, { + "topic" : "OUTPUT", + "key" : "b,2", + "value" : "2,b,0" + }, { + "topic" : "OUTPUT", + "key" : "b,1", + "value" : "1,b,0" + }, { + "topic" : "OUTPUT", + "key" : "a,1", + "value" : "1,a,1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, f2, AS_VALUE(f1) AS F3, AS_VALUE(F2) AS F4, COUNT(*) FROM TEST GROUP BY f2, f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `F1` INTEGER KEY, `F3` INTEGER, `F4` STRING, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.2.0_1608159561540/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.2.0_1608159561540/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_-_copied_into_value_(table-_table)/6.2.0_1608159561540/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_used_in_expression/6.2.0_1608159560748/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_used_in_expression/6.2.0_1608159560748/plan.json new file mode 100644 index 000000000000..8643f5ab9622 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_used_in_expression/6.2.0_1608159560748/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, F1 INTEGER, F2 INTEGER) WITH (FORMAT='DELIMITED', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n (TEST.F1 / TEST.F2) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY TEST.F1, TEST.F2\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F1` INTEGER, `F2` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F1", "F2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F1", "F2" ], + "selectExpressions" : [ "(F1 / F2) AS KSQL_COL_0", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_used_in_expression/6.2.0_1608159560748/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_used_in_expression/6.2.0_1608159560748/spec.json new file mode 100644 index 000000000000..d3d974757cdc --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_used_in_expression/6.2.0_1608159560748/spec.json @@ -0,0 +1,142 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159560748, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `F1` INTEGER, `F2` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` INTEGER", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `F1` INTEGER, `F2` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "fields used in expression", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "4,2" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "9,3" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "9,3" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "4,2", + "value" : "2,1" + }, { + "topic" : "OUTPUT", + "key" : "9,3", + "value" : "3,1" + }, { + "topic" : "OUTPUT", + "key" : "9,3", + "value" : "3,2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 INT) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT F1, F2, f1 / f2, COUNT(*) FROM TEST GROUP BY f1, f2;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` INTEGER", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_used_in_expression/6.2.0_1608159560748/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_used_in_expression/6.2.0_1608159560748/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_fields_used_in_expression/6.2.0_1608159560748/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(stream-_table)/6.2.0_1608159563727/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(stream-_table)/6.2.0_1608159563727/plan.json new file mode 100644 index 000000000000..0b638729e549 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(stream-_table)/6.2.0_1608159563727/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, SOURCE STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n SUBSTRING(TEST.SOURCE, 0, 2) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY SUBSTRING(TEST.SOURCE, 0, 2)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `SOURCE` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "SOURCE AS SOURCE", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "SUBSTRING(SOURCE, 0, 2)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "SOURCE", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(stream-_table)/6.2.0_1608159563727/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(stream-_table)/6.2.0_1608159563727/spec.json new file mode 100644 index 000000000000..3cc5fe4006a1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(stream-_table)/6.2.0_1608159563727/spec.json @@ -0,0 +1,166 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159563727, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `SOURCE` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `SOURCE` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "function (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "some string" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "another string" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "some string again" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "another string again" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "some other string" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "the final string" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "so", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "an", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "so", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "an", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "so", + "value" : "3" + }, { + "topic" : "OUTPUT", + "key" : "th", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT SUBSTRING(source, 0, 2), COUNT(*) FROM TEST GROUP BY SUBSTRING(source, 0, 2);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(stream-_table)/6.2.0_1608159563727/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(stream-_table)/6.2.0_1608159563727/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(stream-_table)/6.2.0_1608159563727/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.2.0_1608159563845/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.2.0_1608159563845/plan.json new file mode 100644 index 000000000000..2623176ff4db --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.2.0_1608159563845/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, USER INTEGER, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n SUBSTRING(TEST.REGION, 7, 2) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY SUBSTRING(TEST.REGION, 7, 2)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "REGION AS REGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "SUBSTRING(REGION, 7, 2)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.2.0_1608159563845/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.2.0_1608159563845/spec.json new file mode 100644 index 000000000000..07896c0f764c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.2.0_1608159563845/spec.json @@ -0,0 +1,170 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159563845, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `REGION` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `REGION` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "function (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,prefixr0" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,prefixr1" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3,prefixr0" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : null + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,prefixr0" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT SUBSTRING(region, 7, 2), COUNT(*) FROM TEST GROUP BY SUBSTRING(region, 7, 2);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.2.0_1608159563845/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.2.0_1608159563845/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_(table-_table)/6.2.0_1608159563845/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_group_by_column_used_in_non-aggregate_function_in_having/6.2.0_1608159558770/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_group_by_column_used_in_non-aggregate_function_in_having/6.2.0_1608159558770/plan.json new file mode 100644 index 000000000000..9dc103a0170b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_group_by_column_used_in_non-aggregate_function_in_having/6.2.0_1608159558770/plan.json @@ -0,0 +1,190 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID INTEGER KEY, COL1 STRING, COL2 STRING, COL3 STRING) WITH (FORMAT='json', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INITCAP(INPUT.COL1) G1,\n INPUT.COL2 G2,\n TRIM(INPUT.COL3) G3,\n COUNT(*) KSQL_COL_0\nFROM INPUT INPUT\nGROUP BY INITCAP(INPUT.COL1), INPUT.COL2, TRIM(INPUT.COL3)\nHAVING (SUBSTRING(TRIM(INPUT.COL3), 1, 4) = 'teen')\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`G1` STRING KEY, `G2` STRING KEY, `G3` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "COL1 AS COL1", "COL2 AS COL2", "COL3 AS COL3", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "INITCAP(COL1)", "COL2", "TRIM(COL3)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "COL1", "COL2", "COL3", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "filterExpression" : "(SUBSTRING(TRIM(COL3), 1, 4) = 'teen')" + }, + "keyColumnNames" : [ "G1", "G2", "G3" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_group_by_column_used_in_non-aggregate_function_in_having/6.2.0_1608159558770/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_group_by_column_used_in_non-aggregate_function_in_having/6.2.0_1608159558770/spec.json new file mode 100644 index 000000000000..4d49c8420714 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_group_by_column_used_in_non-aggregate_function_in_having/6.2.0_1608159558770/spec.json @@ -0,0 +1,172 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159558770, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL2` STRING KEY, `KSQL_COL_1` STRING KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL2` STRING KEY, `KSQL_COL_1` STRING KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`G1` STRING KEY, `G2` STRING KEY, `G3` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "function group by column used in non-aggregate function in having", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "col1" : "smells", + "col2" : "like", + "col3" : "teen spirit" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "col1" : "the", + "col2" : "man who", + "col3" : "stole the world" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "col1" : "smells", + "col2" : "like", + "col3" : "spring" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "col1" : "smells", + "col2" : "like", + "col3" : " teen spirit " + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "G1" : "Smells", + "G2" : "like", + "G3" : "teen spirit" + }, + "value" : { + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "G1" : "Smells", + "G2" : "like", + "G3" : "teen spirit" + }, + "value" : { + "KSQL_COL_0" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', format='json');", "CREATE TABLE OUTPUT AS SELECT INITCAP(COL1) AS G1, COL2 AS G2, trim(COL3) AS G3, COUNT(*) FROM input GROUP BY INITCAP(col1), col2, trim(col3) HAVING substring(trim(col3),1,4) = 'teen';" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`G1` STRING KEY, `G2` STRING KEY, `G3` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_group_by_column_used_in_non-aggregate_function_in_having/6.2.0_1608159558770/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_group_by_column_used_in_non-aggregate_function_in_having/6.2.0_1608159558770/topology new file mode 100644 index 000000000000..b08ea0de8518 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_group_by_column_used_in_non-aggregate_function_in_having/6.2.0_1608159558770/topology @@ -0,0 +1,49 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000014 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000014 (stores: []) + --> KSTREAM-SINK-0000000015 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000015 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000014 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_in_group-by_and_nested_function_in_select/6.2.0_1608159558505/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_in_group-by_and_nested_function_in_select/6.2.0_1608159558505/plan.json new file mode 100644 index 000000000000..667d3034c74b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_in_group-by_and_nested_function_in_select/6.2.0_1608159558505/plan.json @@ -0,0 +1,183 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID INTEGER KEY, COL1 STRING, COL2 STRING, COL3 STRING) WITH (FORMAT='json', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INITCAP(INPUT.COL1) G1,\n INPUT.COL2 G2,\n TRIM(INPUT.COL3) G3,\n CONCAT(INITCAP(INPUT.COL1), INPUT.COL2, TRIM(INPUT.COL3)) FOO,\n COUNT(*) KSQL_COL_0\nFROM INPUT INPUT\nGROUP BY INITCAP(INPUT.COL1), INPUT.COL2, TRIM(INPUT.COL3)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`G1` STRING KEY, `G2` STRING KEY, `G3` STRING KEY, `FOO` STRING, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "COL1 AS COL1", "COL2 AS COL2", "COL3 AS COL3", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "INITCAP(COL1)", "COL2", "TRIM(COL3)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "COL1", "COL2", "COL3", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "G1", "G2", "G3" ], + "selectExpressions" : [ "CONCAT(INITCAP(COL1), COL2, TRIM(COL3)) AS FOO", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_in_group-by_and_nested_function_in_select/6.2.0_1608159558505/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_in_group-by_and_nested_function_in_select/6.2.0_1608159558505/spec.json new file mode 100644 index 000000000000..9153edf77f05 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_in_group-by_and_nested_function_in_select/6.2.0_1608159558505/spec.json @@ -0,0 +1,196 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159558505, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL2` STRING KEY, `KSQL_COL_1` STRING KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL2` STRING KEY, `KSQL_COL_1` STRING KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`G1` STRING KEY, `G2` STRING KEY, `G3` STRING KEY, `FOO` STRING, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "function in group-by and nested function in select", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "col1" : "smells", + "col2" : "like", + "col3" : "teen spirit" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "col1" : "the", + "col2" : "man who", + "col3" : "stole the world" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "col1" : "smells", + "col2" : "like", + "col3" : "spring" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "col1" : "smells", + "col2" : "like", + "col3" : " teen spirit " + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "G1" : "Smells", + "G2" : "like", + "G3" : "teen spirit" + }, + "value" : { + "FOO" : "Smellsliketeen spirit", + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "G1" : "The", + "G2" : "man who", + "G3" : "stole the world" + }, + "value" : { + "FOO" : "Theman whostole the world", + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "G1" : "Smells", + "G2" : "like", + "G3" : "spring" + }, + "value" : { + "FOO" : "Smellslikespring", + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "G1" : "Smells", + "G2" : "like", + "G3" : "teen spirit" + }, + "value" : { + "FOO" : "Smellsliketeen spirit", + "KSQL_COL_0" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', format='json');", "CREATE TABLE OUTPUT AS SELECT INITCAP(COL1) AS G1, COL2 AS G2, TRIM(COL3) AS G3, concat(initcap(col1), col2, trim(col3)) AS foo, COUNT(*) FROM input GROUP BY INITCAP(col1), col2, TRIM(col3);" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`G1` STRING KEY, `G2` STRING KEY, `G3` STRING KEY, `FOO` STRING, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_in_group-by_and_nested_function_in_select/6.2.0_1608159558505/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_in_group-by_and_nested_function_in_select/6.2.0_1608159558505/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_function_in_group-by_and_nested_function_in_select/6.2.0_1608159558505/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_group_by_column_in_nested_non-aggregate_function_in_select/6.2.0_1608159558628/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_group_by_column_in_nested_non-aggregate_function_in_select/6.2.0_1608159558628/plan.json new file mode 100644 index 000000000000..f61a52d4fbf6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_group_by_column_in_nested_non-aggregate_function_in_select/6.2.0_1608159558628/plan.json @@ -0,0 +1,183 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID INTEGER KEY, COL1 STRING, COL2 STRING, COL3 STRING) WITH (FORMAT='json', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INITCAP(INPUT.COL1) G1,\n INPUT.COL2 G2,\n INPUT.COL3 G3,\n CONCAT(INITCAP(INPUT.COL1), INPUT.COL2, TRIM(INPUT.COL3)) FOO,\n COUNT(*) KSQL_COL_0\nFROM INPUT INPUT\nGROUP BY INITCAP(INPUT.COL1), INPUT.COL2, INPUT.COL3\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`G1` STRING KEY, `G2` STRING KEY, `G3` STRING KEY, `FOO` STRING, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "COL1 AS COL1", "COL2 AS COL2", "COL3 AS COL3", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "INITCAP(COL1)", "COL2", "COL3" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "COL1", "COL2", "COL3", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "G1", "G2", "G3" ], + "selectExpressions" : [ "CONCAT(INITCAP(COL1), COL2, TRIM(COL3)) AS FOO", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_group_by_column_in_nested_non-aggregate_function_in_select/6.2.0_1608159558628/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_group_by_column_in_nested_non-aggregate_function_in_select/6.2.0_1608159558628/spec.json new file mode 100644 index 000000000000..a71aad7eedd1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_group_by_column_in_nested_non-aggregate_function_in_select/6.2.0_1608159558628/spec.json @@ -0,0 +1,196 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159558628, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL2` STRING KEY, `COL3` STRING KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL2` STRING KEY, `COL3` STRING KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`G1` STRING KEY, `G2` STRING KEY, `G3` STRING KEY, `FOO` STRING, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "group by column in nested non-aggregate function in select", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "col1" : "smells", + "col2" : "like", + "col3" : "teen spirit" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "col1" : "the", + "col2" : "man who", + "col3" : "stole the world" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "col1" : "smells", + "col2" : "like", + "col3" : "spring" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "col1" : "smells", + "col2" : "like", + "col3" : " teen spirit " + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "G1" : "Smells", + "G2" : "like", + "G3" : "teen spirit" + }, + "value" : { + "FOO" : "Smellsliketeen spirit", + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "G1" : "The", + "G2" : "man who", + "G3" : "stole the world" + }, + "value" : { + "FOO" : "Theman whostole the world", + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "G1" : "Smells", + "G2" : "like", + "G3" : "spring" + }, + "value" : { + "FOO" : "Smellslikespring", + "KSQL_COL_0" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "G1" : "Smells", + "G2" : "like", + "G3" : " teen spirit " + }, + "value" : { + "FOO" : "Smellsliketeen spirit", + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', format='json');", "CREATE TABLE OUTPUT AS SELECT INITCAP(COL1) AS G1, COL2 AS G2, COL3 AS G3, concat(initcap(col1), col2, trim(col3)) AS foo, COUNT(*) FROM input GROUP BY INITCAP(col1), col2, col3;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `COL1` STRING, `COL2` STRING, `COL3` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`G1` STRING KEY, `G2` STRING KEY, `G3` STRING KEY, `FOO` STRING, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_group_by_column_in_nested_non-aggregate_function_in_select/6.2.0_1608159558628/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_group_by_column_in_nested_non-aggregate_function_in_select/6.2.0_1608159558628/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_group_by_column_in_nested_non-aggregate_function_in_select/6.2.0_1608159558628/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_field_(stream-_table)/6.2.0_1608159560529/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_field_(stream-_table)/6.2.0_1608159560529/plan.json new file mode 100644 index 000000000000..47d4237c6542 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_field_(stream-_table)/6.2.0_1608159560529/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, ID INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.ID ID,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ID` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_field_(stream-_table)/6.2.0_1608159560529/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_field_(stream-_table)/6.2.0_1608159560529/spec.json new file mode 100644 index 000000000000..4b857d42b78c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_field_(stream-_table)/6.2.0_1608159560529/spec.json @@ -0,0 +1,188 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159560529, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`ID` INTEGER KEY, `ID` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ID` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` INTEGER KEY, `ID` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ID` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "int field (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "a", + "value" : { + "ID" : 1 + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "b", + "value" : { + "ID" : 2 + }, + "timestamp" : 2 + }, { + "topic" : "test_topic", + "key" : "c", + "value" : { + "ID" : 1 + }, + "timestamp" : 3 + }, { + "topic" : "test_topic", + "key" : "d", + "value" : { + "ID" : 2 + }, + "timestamp" : 4 + }, { + "topic" : "test_topic", + "key" : "e", + "value" : { + "ID" : 1 + }, + "timestamp" : 5 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "KSQL_COL_0" : 1 + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "KSQL_COL_0" : 1 + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "KSQL_COL_0" : 2 + }, + "timestamp" : 3 + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "KSQL_COL_0" : 2 + }, + "timestamp" : 4 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "KSQL_COL_0" : 3 + }, + "timestamp" : 5 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, ID INT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT ID, COUNT(*) FROM TEST GROUP BY ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_field_(stream-_table)/6.2.0_1608159560529/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_field_(stream-_table)/6.2.0_1608159560529/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_field_(stream-_table)/6.2.0_1608159560529/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.2.0_1608159563999/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.2.0_1608159563999/plan.json new file mode 100644 index 000000000000..b8bfb0087f4a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.2.0_1608159563999/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEN(TEST.REGION) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY LEN(TEST.REGION)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "REGION AS REGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "LEN(REGION)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.2.0_1608159563999/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.2.0_1608159563999/spec.json new file mode 100644 index 000000000000..43e3dd1e0a02 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.2.0_1608159563999/spec.json @@ -0,0 +1,170 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159563999, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `REGION` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `REGION` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "int function (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "usa" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "eu" + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "usa" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "usa" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT LEN(region), COUNT(*) FROM TEST GROUP BY LEN(region);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.2.0_1608159563999/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.2.0_1608159563999/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_function_(table-_table)/6.2.0_1608159563999/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_json_field_(stream-_table)/6.2.0_1608159564250/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_json_field_(stream-_table)/6.2.0_1608159564250/plan.json new file mode 100644 index 000000000000..3a76bd6b0b46 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_json_field_(stream-_table)/6.2.0_1608159564250/plan.json @@ -0,0 +1,180 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (DATA STRUCT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`DATA` STRUCT<`FIELD` INTEGER>", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA->FIELD FIELD,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.DATA->FIELD\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`FIELD` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`DATA` STRUCT<`FIELD` INTEGER>" + }, + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "DATA->FIELD" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "FIELD" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_json_field_(stream-_table)/6.2.0_1608159564250/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_json_field_(stream-_table)/6.2.0_1608159564250/spec.json new file mode 100644 index 000000000000..03955a8e0005 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_json_field_(stream-_table)/6.2.0_1608159564250/spec.json @@ -0,0 +1,172 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159564250, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`FIELD` INTEGER KEY, `DATA` STRUCT<`FIELD` INTEGER>, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`DATA` STRUCT<`FIELD` INTEGER>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`FIELD` INTEGER KEY, `DATA` STRUCT<`FIELD` INTEGER>, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`FIELD` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "int json field (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : { + "field" : 1 + } + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : { + "field" : 2 + } + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : { } + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : { + "field" : 1 + } + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : { } + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (data STRUCT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT data->field, COUNT(*) AS COUNT FROM TEST GROUP BY data->field;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`FIELD` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`DATA` STRUCT<`FIELD` INTEGER>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_json_field_(stream-_table)/6.2.0_1608159564250/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_json_field_(stream-_table)/6.2.0_1608159564250/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_int_json_field_(stream-_table)/6.2.0_1608159564250/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_json_field_(stream-_table)/6.2.0_1608159564132/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_json_field_(stream-_table)/6.2.0_1608159564132/plan.json new file mode 100644 index 000000000000..3f9c8a313c7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_json_field_(stream-_table)/6.2.0_1608159564132/plan.json @@ -0,0 +1,180 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (DATA STRUCT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`DATA` STRUCT<`FIELD` STRING>", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA->FIELD FIELD,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.DATA->FIELD\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`FIELD` STRING KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`DATA` STRUCT<`FIELD` STRING>" + }, + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "DATA->FIELD" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "FIELD" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_json_field_(stream-_table)/6.2.0_1608159564132/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_json_field_(stream-_table)/6.2.0_1608159564132/spec.json new file mode 100644 index 000000000000..0e752f703d47 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_json_field_(stream-_table)/6.2.0_1608159564132/spec.json @@ -0,0 +1,172 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159564132, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`FIELD` STRING KEY, `DATA` STRUCT<`FIELD` STRING>, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`DATA` STRUCT<`FIELD` STRING>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`FIELD` STRING KEY, `DATA` STRUCT<`FIELD` STRING>, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`FIELD` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "json field (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : { + "field" : "Something" + } + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : { + "field" : "Something Else" + } + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : { } + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : { + "field" : "Something" + } + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : { } + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "Something", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "Something Else", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "Something", + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (data STRUCT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT data->field AS FIELD, COUNT(*) AS COUNT FROM TEST GROUP BY data->field;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`FIELD` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`DATA` STRUCT<`FIELD` STRING>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_json_field_(stream-_table)/6.2.0_1608159564132/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_json_field_(stream-_table)/6.2.0_1608159564132/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_json_field_(stream-_table)/6.2.0_1608159564132/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(stream-_table)/6.2.0_1608159566065/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(stream-_table)/6.2.0_1608159566065/plan.json new file mode 100644 index 000000000000..5df01b505cb7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(stream-_table)/6.2.0_1608159566065/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA DATA,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "DATA" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(stream-_table)/6.2.0_1608159566065/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(stream-_table)/6.2.0_1608159566065/spec.json new file mode 100644 index 000000000000..9dfd3e72bbb6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(stream-_table)/6.2.0_1608159566065/spec.json @@ -0,0 +1,160 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159566065, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "missing matching projection field (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d2" + }, + "timestamp" : 2 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "DATA" : "d1" + }, + "timestamp" : 3 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 1 + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_0" : 1 + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "KSQL_COL_0" : 2 + }, + "timestamp" : 3 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT data, COUNT(*) FROM TEST GROUP BY data;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(stream-_table)/6.2.0_1608159566065/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(stream-_table)/6.2.0_1608159566065/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(stream-_table)/6.2.0_1608159566065/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.2.0_1608159566171/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.2.0_1608159566171/plan.json new file mode 100644 index 000000000000..f8cbbd1487e9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.2.0_1608159566171/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F2" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.2.0_1608159566171/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.2.0_1608159566171/spec.json new file mode 100644 index 000000000000..963425b541de --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.2.0_1608159566171/spec.json @@ -0,0 +1,174 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159566171, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F2` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F2` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "missing matching projection field (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,b" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : null + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "a", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "b", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "a", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "b", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "b", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "b", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "a", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT F2, COUNT(*) FROM TEST GROUP BY f2;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.2.0_1608159566171/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.2.0_1608159566171/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_missing_matching_projection_field_(table-_table)/6.2.0_1608159566171/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions/6.2.0_1608159559110/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions/6.2.0_1608159559110/plan.json new file mode 100644 index 000000000000..48bede70ac8d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions/6.2.0_1608159559110/plan.json @@ -0,0 +1,183 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (F1 INTEGER KEY, F2 INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F2 F2,\n TEST.F1 F1,\n (TEST.F2 + TEST.F1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY TEST.F1, TEST.F2\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`F1` INTEGER KEY, `F2` INTEGER" + }, + "keyColumnNames" : [ "F1" ], + "selectExpressions" : [ "F2 AS F2", "F1 AS F1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "F1", "F2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "F2", "F1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F1", "F2" ], + "selectExpressions" : [ "(F2 + F1) AS KSQL_COL_0", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions/6.2.0_1608159559110/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions/6.2.0_1608159559110/spec.json new file mode 100644 index 000000000000..241f2b904cc5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions/6.2.0_1608159559110/spec.json @@ -0,0 +1,200 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159559110, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `F2` INTEGER, `F1` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `F2` INTEGER, `F1` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "multiple expressions", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "f2" : "2" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "f2" : "4" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "f2" : "2" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "f2" : "4" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "f2" : "1" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "F1" : 1, + "F2" : 2 + }, + "value" : { + "KSQL_COL_0" : 3, + "KSQL_COL_1" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 2, + "F2" : 4 + }, + "value" : { + "KSQL_COL_0" : 6, + "KSQL_COL_1" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 1, + "F2" : 2 + }, + "value" : { + "KSQL_COL_0" : 3, + "KSQL_COL_1" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 2, + "F2" : 4 + }, + "value" : { + "KSQL_COL_0" : 6, + "KSQL_COL_1" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 2, + "F2" : 1 + }, + "value" : { + "KSQL_COL_0" : 3, + "KSQL_COL_1" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (f1 INT KEY, f2 INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT f2, f1, f2+f1, COUNT(*) FROM TEST GROUP BY f1, f2;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions/6.2.0_1608159559110/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions/6.2.0_1608159559110/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions/6.2.0_1608159559110/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_table_aggregate/6.2.0_1608159559471/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_table_aggregate/6.2.0_1608159559471/plan.json new file mode 100644 index 000000000000..175bd1a29305 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_table_aggregate/6.2.0_1608159559471/plan.json @@ -0,0 +1,184 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (ID INTEGER PRIMARY KEY, VALUE INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n 1 K,\n INPUT.VALUE VALUE,\n COUNT(1) ID\nFROM INPUT INPUT\nGROUP BY INPUT.VALUE, 1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`VALUE` INTEGER KEY, `K` INTEGER KEY, `ID` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `VALUE` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "VALUE AS VALUE", "1 AS KSQL_INTERNAL_COL_1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "VALUE", "1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "VALUE" ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_1)" ] + }, + "keyColumnNames" : [ "VALUE", "K" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS ID" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_table_aggregate/6.2.0_1608159559471/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_table_aggregate/6.2.0_1608159559471/spec.json new file mode 100644 index 000000000000..a4229cad66a1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_table_aggregate/6.2.0_1608159559471/spec.json @@ -0,0 +1,174 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159559471, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`VALUE` INTEGER KEY, `KSQL_COL_0` INTEGER KEY, `VALUE` INTEGER, `KSQL_INTERNAL_COL_1` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`VALUE` INTEGER KEY, `KSQL_COL_0` INTEGER KEY, `VALUE` INTEGER, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`VALUE` INTEGER KEY, `K` INTEGER KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "multiple expressions - table aggregate", + "inputs" : [ { + "topic" : "test_topic", + "key" : 10, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 1666, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 98, + "value" : { + "VALUE" : 0 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "VALUE" : 0, + "K" : 1 + }, + "value" : { + "ID" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "VALUE" : 0, + "K" : 1 + }, + "value" : { + "ID" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : { + "VALUE" : 0, + "K" : 1 + }, + "value" : { + "ID" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (ID INT PRIMARY KEY, VALUE INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT as SELECT 1 as k, value, count(1) AS ID FROM INPUT group by value, 1;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`VALUE` INTEGER KEY, `K` INTEGER KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_table_aggregate/6.2.0_1608159559471/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_table_aggregate/6.2.0_1608159559471/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_table_aggregate/6.2.0_1608159559471/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_windowed/6.2.0_1608159559337/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_windowed/6.2.0_1608159559337/plan.json new file mode 100644 index 000000000000..5545a1522d79 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_windowed/6.2.0_1608159559337/plan.json @@ -0,0 +1,188 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (F1 INTEGER KEY, F2 INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F2 F2,\n TEST.F1 F1,\n (TEST.F2 + TEST.F1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nWINDOW TUMBLING ( SIZE 1 SECONDS ) \nGROUP BY TEST.F1, TEST.F2\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamWindowedAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`F1` INTEGER KEY, `F2` INTEGER" + }, + "keyColumnNames" : [ "F1" ], + "selectExpressions" : [ "F2 AS F2", "F1 AS F1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "F1", "F2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "F2", "F1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ], + "windowExpression" : " TUMBLING ( SIZE 1 SECONDS ) " + }, + "keyColumnNames" : [ "F1", "F2" ], + "selectExpressions" : [ "(F2 + F1) AS KSQL_COL_0", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_windowed/6.2.0_1608159559337/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_windowed/6.2.0_1608159559337/spec.json new file mode 100644 index 000000000000..246a53534f75 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_windowed/6.2.0_1608159559337/spec.json @@ -0,0 +1,239 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159559337, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `F2` INTEGER, `F1` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `F2` INTEGER, `F1` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "JSON", + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "multiple expressions - windowed", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "f2" : "2" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "f2" : "4" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "f2" : "2" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "f2" : "4" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "f2" : "1" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "F1" : 1, + "F2" : 2 + }, + "value" : { + "KSQL_COL_0" : 3, + "KSQL_COL_1" : 1 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 2, + "F2" : 4 + }, + "value" : { + "KSQL_COL_0" : 6, + "KSQL_COL_1" : 1 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 1, + "F2" : 2 + }, + "value" : { + "KSQL_COL_0" : 3, + "KSQL_COL_1" : 2 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 2, + "F2" : 4 + }, + "value" : { + "KSQL_COL_0" : 6, + "KSQL_COL_1" : 2 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 2, + "F2" : 1 + }, + "value" : { + "KSQL_COL_0" : 3, + "KSQL_COL_1" : 1 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (f1 INT KEY, f2 INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT f2, f1, f2+f1, COUNT(*) FROM TEST WINDOW TUMBLING (SIZE 1 SECOND) GROUP BY f1, f2;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "JSON", + "windowType" : "TUMBLING", + "windowSize" : 1000 + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_windowed/6.2.0_1608159559337/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_windowed/6.2.0_1608159559337/topology new file mode 100644 index 000000000000..bdf02e6d2f71 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_-_windowed/6.2.0_1608159559337/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Aggregate-WindowSelect + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Aggregate-WindowSelect (stores: []) + --> Aggregate-Project + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- Aggregate-Aggregate-WindowSelect + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000013 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000012 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_nulls/6.2.0_1608159560052/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_nulls/6.2.0_1608159560052/plan.json new file mode 100644 index 000000000000..35f2b9cad1ac --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_nulls/6.2.0_1608159560052/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n BAD_UDF(TEST.DATA) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY BAD_UDF(TEST.DATA)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "BAD_UDF(DATA)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_nulls/6.2.0_1608159560052/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_nulls/6.2.0_1608159560052/spec.json new file mode 100644 index 000000000000..97dcc1ce6e25 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_nulls/6.2.0_1608159560052/spec.json @@ -0,0 +1,160 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159560052, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "multiple expressions with nulls", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "data" : "d1" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_1" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_1" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT bad_udf(DATA), COUNT(*) FROM TEST GROUP BY bad_udf(DATA);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_nulls/6.2.0_1608159560052/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_nulls/6.2.0_1608159560052/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_nulls/6.2.0_1608159560052/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_struct_field_and_other_expression/6.2.0_1608159559211/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_struct_field_and_other_expression/6.2.0_1608159559211/plan.json new file mode 100644 index 000000000000..b0546eca3148 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_struct_field_and_other_expression/6.2.0_1608159559211/plan.json @@ -0,0 +1,183 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (F1 INTEGER KEY, F2 INTEGER, ADDRESS STRUCT) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.ADDRESS->TOWN TOWN,\n TEST.F1 F1,\n ((2 * TEST.F2) + TEST.F1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1,\n (2 * TEST.F2) KSQL_COL_2\nFROM TEST TEST\nGROUP BY TEST.F1, TEST.ADDRESS->TOWN, (2 * TEST.F2)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F1` INTEGER KEY, `TOWN` STRING KEY, `KSQL_COL_2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`F1` INTEGER KEY, `F2` INTEGER, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>" + }, + "keyColumnNames" : [ "F1" ], + "selectExpressions" : [ "ADDRESS AS ADDRESS", "F1 AS F1", "F2 AS F2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "F1", "ADDRESS->TOWN", "(2 * F2)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ADDRESS", "F1", "F2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F1", "TOWN", "KSQL_COL_2" ], + "selectExpressions" : [ "((2 * F2) + F1) AS KSQL_COL_0", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_struct_field_and_other_expression/6.2.0_1608159559211/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_struct_field_and_other_expression/6.2.0_1608159559211/spec.json new file mode 100644 index 000000000000..e4f0d58c7270 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_struct_field_and_other_expression/6.2.0_1608159559211/spec.json @@ -0,0 +1,225 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159559211, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F1` INTEGER KEY, `TOWN` STRING KEY, `KSQL_COL_0` INTEGER KEY, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>, `F1` INTEGER, `F2` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`F1` INTEGER KEY, `F2` INTEGER, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F1` INTEGER KEY, `TOWN` STRING KEY, `KSQL_COL_0` INTEGER KEY, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>, `F1` INTEGER, `F2` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F1` INTEGER KEY, `TOWN` STRING KEY, `KSQL_COL_2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "multiple expressions with struct field and other expression", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "f2" : "2", + "ADDRESS" : { + "STREET" : "1st Street", + "Town" : "Oxford" + } + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "f2" : "4", + "ADDRESS" : { + "STREET" : "1st Street", + "Town" : "London" + } + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "f2" : "2", + "ADDRESS" : { + "STREET" : "1st Street", + "Town" : "Oxford" + } + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "f2" : "4", + "ADDRESS" : { + "STREET" : "1st Street", + "Town" : "London" + } + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "f2" : "1", + "ADDRESS" : { + "STREET" : "1st Street", + "Town" : "Oxford" + } + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "F1" : 1, + "TOWN" : "Oxford", + "KSQL_COL_2" : 4 + }, + "value" : { + "KSQL_COL_0" : 5, + "KSQL_COL_1" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 2, + "TOWN" : "London", + "KSQL_COL_2" : 8 + }, + "value" : { + "KSQL_COL_0" : 10, + "KSQL_COL_1" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 1, + "TOWN" : "Oxford", + "KSQL_COL_2" : 4 + }, + "value" : { + "KSQL_COL_0" : 5, + "KSQL_COL_1" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 2, + "TOWN" : "London", + "KSQL_COL_2" : 8 + }, + "value" : { + "KSQL_COL_0" : 10, + "KSQL_COL_1" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : { + "F1" : 2, + "TOWN" : "Oxford", + "KSQL_COL_2" : 2 + }, + "value" : { + "KSQL_COL_0" : 4, + "KSQL_COL_1" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (f1 INT KEY, f2 INT, address STRUCT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT address->town, f1, 2*f2+f1, COUNT(*), 2*f2 FROM TEST GROUP BY f1, address->town, 2*f2;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F1` INTEGER KEY, `TOWN` STRING KEY, `KSQL_COL_2` INTEGER KEY, `KSQL_COL_0` INTEGER, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`F1` INTEGER KEY, `F2` INTEGER, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_struct_field_and_other_expression/6.2.0_1608159559211/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_struct_field_and_other_expression/6.2.0_1608159559211/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_multiple_expressions_with_struct_field_and_other_expression/6.2.0_1608159559211/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_non-KAFKA_key_format/6.2.0_1608159568407/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_non-KAFKA_key_format/6.2.0_1608159568407/plan.json new file mode 100644 index 000000000000..765e1d13c505 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_non-KAFKA_key_format/6.2.0_1608159568407/plan.json @@ -0,0 +1,187 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, VAL BOOLEAN) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `VAL` BOOLEAN", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.VAL VAL,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.VAL\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `VAL` BOOLEAN" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "VAL AS VAL", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "VAL" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "VAL", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "VAL" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_non-KAFKA_key_format/6.2.0_1608159568407/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_non-KAFKA_key_format/6.2.0_1608159568407/spec.json new file mode 100644 index 000000000000..8e148f000261 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_non-KAFKA_key_format/6.2.0_1608159568407/spec.json @@ -0,0 +1,162 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159568407, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`VAL` BOOLEAN KEY, `VAL` BOOLEAN, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `VAL` BOOLEAN", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`VAL` BOOLEAN KEY, `VAL` BOOLEAN, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "non-KAFKA key format", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VAL" : true + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "VAL" : false + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "VAL" : true + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : false, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, VAL BOOLEAN) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT VAL, COUNT() AS COUNT FROM TEST GROUP BY VAL;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `VAL` BOOLEAN", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_non-KAFKA_key_format/6.2.0_1608159568407/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_non-KAFKA_key_format/6.2.0_1608159568407/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_non-KAFKA_key_format/6.2.0_1608159568407/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.2.0_1608159567375/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.2.0_1608159567375/plan.json new file mode 100644 index 000000000000..998b8d583f49 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.2.0_1608159567375/plan.json @@ -0,0 +1,248 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, TOTAL INTEGER) WITH (KAFKA_TOPIC='T1', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, TOTAL INTEGER) WITH (KAFKA_TOPIC='T2', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T1.ID T1_ID,\n SUM((T1.TOTAL + (CASE WHEN (T2.TOTAL IS NULL) THEN 0 ELSE T2.TOTAL END))) SUM\nFROM T1 T1\nLEFT OUTER JOIN T2 T2 ON ((T1.ID = T2.ID))\nGROUP BY T1.ID\nHAVING (COUNT(1) > 0)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T1_ID` BIGINT KEY, `SUM` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "TOTAL AS T1_TOTAL", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "TOTAL AS T2_TOTAL", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T1_ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "T1_ID AS T1_ID", "T1_TOTAL AS T1_TOTAL", "T2_TOTAL AS T2_TOTAL", "(T1_TOTAL + (CASE WHEN (T2_TOTAL IS NULL) THEN 0 ELSE T2_TOTAL END)) AS KSQL_INTERNAL_COL_3", "1 AS KSQL_INTERNAL_COL_4" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "T1_ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "T1_ID", "T1_TOTAL", "T2_TOTAL" ], + "aggregationFunctions" : [ "SUM(KSQL_INTERNAL_COL_3)", "COUNT(KSQL_INTERNAL_COL_4)" ] + }, + "filterExpression" : "(KSQL_AGG_VARIABLE_1 > 0)" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS SUM" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.2.0_1608159567375/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.2.0_1608159567375/spec.json new file mode 100644 index 000000000000..3c95e4cd2242 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.2.0_1608159567375/spec.json @@ -0,0 +1,380 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159567375, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`T1_ID` BIGINT KEY, `T1_ID` BIGINT, `T1_TOTAL` INTEGER, `T2_TOTAL` INTEGER, `KSQL_INTERNAL_COL_3` INTEGER, `KSQL_INTERNAL_COL_4` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`T1_ID` BIGINT KEY, `T1_ID` BIGINT, `T1_TOTAL` INTEGER, `T2_TOTAL` INTEGER, `KSQL_AGG_VARIABLE_0` INTEGER, `KSQL_AGG_VARIABLE_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T1_ID` BIGINT KEY, `SUM` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "on join", + "inputs" : [ { + "topic" : "T1", + "key" : 0, + "value" : { + "total" : 100 + } + }, { + "topic" : "T1", + "key" : 1, + "value" : { + "total" : 101 + } + }, { + "topic" : "T2", + "key" : 0, + "value" : { + "total" : 5 + } + }, { + "topic" : "T2", + "key" : 1, + "value" : { + "total" : 10 + } + }, { + "topic" : "T2", + "key" : 0, + "value" : { + "total" : 20 + } + }, { + "topic" : "T2", + "key" : 0, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 100 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "SUM" : 101 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 105 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "SUM" : 111 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 120 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 100 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T1", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T2", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE t1 (ID BIGINT PRIMARY KEY, TOTAL integer) WITH (kafka_topic='T1', value_format='AVRO');", "CREATE TABLE t2 (ID BIGINT PRIMARY KEY, TOTAL integer) WITH (kafka_topic='T2', value_format='AVRO');", "CREATE TABLE OUTPUT AS SELECT t1.ID, SUM(t1.total + CASE WHEN t2.total IS NULL THEN 0 ELSE t2.total END) as SUM FROM T1 LEFT JOIN T2 ON (t1.ID = t2.ID) GROUP BY t1.ID HAVING COUNT(1) > 0;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`T1_ID` BIGINT KEY, `SUM` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "T1", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "T2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "T1", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "T1_ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "T1_TOTAL", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "T2_TOTAL", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "KSQL_INTERNAL_COL_3", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "KSQL_INTERNAL_COL_4", + "type" : [ "null", "int" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "T1_ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "T1_TOTAL", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "T2_TOTAL", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_1", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "SUM", + "type" : [ "null", "int" ], + "default" : null + } ] + } + }, { + "name" : "T2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.2.0_1608159567375/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.2.0_1608159567375/topology new file mode 100644 index 000000000000..69967bd39a24 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_on_join/6.2.0_1608159567375/topology @@ -0,0 +1,78 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [T1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [T2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-FILTER-0000000016 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000018 + <-- KTABLE-FILTER-0000000016 + Sink: KSTREAM-SINK-0000000018 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000019 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000020 + Processor: KTABLE-AGGREGATE-0000000020 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000019 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KTABLE-AGGREGATE-0000000020 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000026 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000026 (stores: []) + --> KSTREAM-SINK-0000000027 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000027 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000026 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_select___where_all_columns_in_group_by/6.2.0_1608159559592/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_select___where_all_columns_in_group_by/6.2.0_1608159559592/plan.json new file mode 100644 index 000000000000..7a334b69c7e2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_select___where_all_columns_in_group_by/6.2.0_1608159559592/plan.json @@ -0,0 +1,183 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, ID2 INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `ID2` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n *,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.ID, TEST.ID2\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ID` INTEGER KEY, `ID2` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `ID2` INTEGER" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "ID2 AS ID2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID", "ID2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "ID2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "ID", "ID2" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_select___where_all_columns_in_group_by/6.2.0_1608159559592/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_select___where_all_columns_in_group_by/6.2.0_1608159559592/spec.json new file mode 100644 index 000000000000..dfe78957b412 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_select___where_all_columns_in_group_by/6.2.0_1608159559592/spec.json @@ -0,0 +1,135 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159559592, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`ID` INTEGER KEY, `ID2` INTEGER KEY, `ID` INTEGER, `ID2` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `ID2` INTEGER", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` INTEGER KEY, `ID2` INTEGER KEY, `ID` INTEGER, `ID2` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ID` INTEGER KEY, `ID2` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "select * where all columns in group by", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : { + "ID2" : 2 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "ID" : 1, + "ID2" : 2 + }, + "value" : { + "KSQL_COL_0" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (id INT KEY, id2 INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT *, COUNT() FROM TEST GROUP BY id, id2;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `ID2` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `ID2` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_select___where_all_columns_in_group_by/6.2.0_1608159559592/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_select___where_all_columns_in_group_by/6.2.0_1608159559592/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_select___where_all_columns_in_group_by/6.2.0_1608159559592/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566830/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566830/plan.json new file mode 100644 index 000000000000..d76b3c722b7a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566830/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, STR STRING, POS INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `STR` STRING, `POS` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n SUBSTRING(TEST.STR, TEST.POS) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY SUBSTRING(TEST.STR, TEST.POS)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `STR` STRING, `POS` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "STR AS STR", "POS AS POS", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "SUBSTRING(STR, POS)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "STR", "POS", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566830/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566830/spec.json new file mode 100644 index 000000000000..bb2609a2cdb2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566830/spec.json @@ -0,0 +1,138 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159566830, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `STR` STRING, `POS` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `STR` STRING, `POS` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `STR` STRING, `POS` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "should exclude any stream row whose single GROUP BY expression resolves to NULL", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "xx,1" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "x," + }, { + "topic" : "test_topic", + "key" : null, + "value" : "xx,1" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "xx", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "xx", + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, str STRING, pos INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT SUBSTRING(str, pos), COUNT() FROM TEST GROUP BY SUBSTRING(str, pos);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `STR` STRING, `POS` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566830/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566830/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566830/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567022/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567022/plan.json new file mode 100644 index 000000000000..a2f919b74716 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567022/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, ID INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n BAD_UDF(TEST.ID) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY BAD_UDF(TEST.ID)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "BAD_UDF(ID)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ID", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567022/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567022/spec.json new file mode 100644 index 000000000000..a0dbc4d179d7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567022/spec.json @@ -0,0 +1,101 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159567022, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `ID` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ID` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `ID` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "should exclude any stream row whose single GROUP BY expression throws", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "1" + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, id INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT BAD_UDF(id), COUNT() FROM TEST GROUP BY BAD_UDF(id);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567022/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567022/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_stream_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567022/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566919/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566919/plan.json new file mode 100644 index 000000000000..337b60865f3e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566919/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, STR STRING, POS INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `STR` STRING, `POS` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n SUBSTRING(TEST.STR, TEST.POS) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY SUBSTRING(TEST.STR, TEST.POS)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `STR` STRING, `POS` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "STR AS STR", "POS AS POS", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "SUBSTRING(STR, POS)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "STR", "POS", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566919/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566919/spec.json new file mode 100644 index 000000000000..32abd64b59f5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566919/spec.json @@ -0,0 +1,146 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159566919, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `STR` STRING, `POS` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `STR` STRING, `POS` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `STR` STRING, `POS` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "should exclude any table row whose single GROUP BY expression resolves to NULL", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "xx,1" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "x," + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "xx,1" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "xx", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "xx", + "value" : "2" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, str STRING, pos INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT SUBSTRING(str, pos), COUNT() FROM TEST GROUP BY SUBSTRING(str, pos);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `STR` STRING, `POS` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566919/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566919/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_resolves_to_NULL/6.2.0_1608159566919/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567097/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567097/plan.json new file mode 100644 index 000000000000..46d2a27eb6a7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567097/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, ID INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n BAD_UDF(TEST.ID) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY BAD_UDF(TEST.ID)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "BAD_UDF(ID)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ID", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567097/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567097/spec.json new file mode 100644 index 000000000000..6e71c6dcb252 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567097/spec.json @@ -0,0 +1,109 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159567097, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `ID` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ID` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `ID` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "should exclude any table row whose single GROUP BY expression throws", + "inputs" : [ { + "topic" : "test_topic", + "key" : "2", + "value" : "1" + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, id INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT BAD_UDF(id), COUNT() FROM TEST GROUP BY BAD_UDF(id);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `ID` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567097/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567097/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_should_exclude_any_table_row_whose_single_GROUP_BY_expression_throws/6.2.0_1608159567097/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(stream-_table)/6.2.0_1608159557507/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(stream-_table)/6.2.0_1608159557507/plan.json new file mode 100644 index 000000000000..da0ad23e2cbb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(stream-_table)/6.2.0_1608159557507/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA NEW_KEY,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "NEW_KEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(stream-_table)/6.2.0_1608159557507/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(stream-_table)/6.2.0_1608159557507/spec.json new file mode 100644 index 000000000000..733b2a2146b2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(stream-_table)/6.2.0_1608159557507/spec.json @@ -0,0 +1,178 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159557507, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "single column with alias (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "data" : "d1" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT DATA AS NEW_KEY, COUNT(*) AS COUNT FROM TEST GROUP BY DATA;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(stream-_table)/6.2.0_1608159557507/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(stream-_table)/6.2.0_1608159557507/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(stream-_table)/6.2.0_1608159557507/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.2.0_1608159557620/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.2.0_1608159557620/plan.json new file mode 100644 index 000000000000..23d1c0ef4ff4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.2.0_1608159557620/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA NEW_KEY,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `DATA` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "NEW_KEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.2.0_1608159557620/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.2.0_1608159557620/spec.json new file mode 100644 index 000000000000..0a65300c0744 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.2.0_1608159557620/spec.json @@ -0,0 +1,186 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159557620, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "single column with alias (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "data" : "d1" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT DATA AS NEW_KEY, COUNT(*) AS COUNT FROM TEST GROUP BY DATA;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.2.0_1608159557620/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.2.0_1608159557620/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_column_with_alias_(table-_table)/6.2.0_1608159557620/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_(stream-_table)/6.2.0_1608159557381/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_(stream-_table)/6.2.0_1608159557381/plan.json new file mode 100644 index 000000000000..8bb9ae75f182 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_(stream-_table)/6.2.0_1608159557381/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEN(TEST.DATA) KSQL_COL_0,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY LEN(TEST.DATA)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "LEN(DATA)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_(stream-_table)/6.2.0_1608159557381/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_(stream-_table)/6.2.0_1608159557381/spec.json new file mode 100644 index 000000000000..0ce007e630b7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_(stream-_table)/6.2.0_1608159557381/spec.json @@ -0,0 +1,178 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159557381, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "single expression (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "data" : "22" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "data" : "333" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "data" : "-2" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "data" : "003" + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "data" : "2-" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, DATA STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT LEN(DATA), COUNT(*) AS COUNT FROM TEST GROUP BY LEN(DATA);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_(stream-_table)/6.2.0_1608159557381/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_(stream-_table)/6.2.0_1608159557381/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_(stream-_table)/6.2.0_1608159557381/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(stream-_table)/6.2.0_1608159557775/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(stream-_table)/6.2.0_1608159557775/plan.json new file mode 100644 index 000000000000..a864b94b763a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(stream-_table)/6.2.0_1608159557775/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEN(TEST.DATA) NEW_KEY,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY LEN(TEST.DATA)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NEW_KEY` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "LEN(DATA)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "NEW_KEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(stream-_table)/6.2.0_1608159557775/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(stream-_table)/6.2.0_1608159557775/spec.json new file mode 100644 index 000000000000..10e2c630a033 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(stream-_table)/6.2.0_1608159557775/spec.json @@ -0,0 +1,178 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159557775, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NEW_KEY` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "single expression with alias (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "data" : "22" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "data" : "333" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "data" : "-2" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "data" : "003" + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "data" : "2-" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT LEN(DATA) AS NEW_KEY, COUNT(*) AS COUNT FROM TEST GROUP BY LEN(DATA);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NEW_KEY` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(stream-_table)/6.2.0_1608159557775/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(stream-_table)/6.2.0_1608159557775/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(stream-_table)/6.2.0_1608159557775/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.2.0_1608159557873/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.2.0_1608159557873/plan.json new file mode 100644 index 000000000000..5bb2b8d83ed9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.2.0_1608159557873/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEN(TEST.DATA) NEW_KEY,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY LEN(TEST.DATA)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NEW_KEY` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `DATA` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "LEN(DATA)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "NEW_KEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.2.0_1608159557873/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.2.0_1608159557873/spec.json new file mode 100644 index 000000000000..4e96755c3896 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.2.0_1608159557873/spec.json @@ -0,0 +1,186 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159557873, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NEW_KEY` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "single expression with alias (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "data" : "22" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "data" : "333" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "data" : "-2" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "data" : "003" + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "data" : "2-" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : { + "COUNT" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT LEN(DATA) AS NEW_KEY, COUNT(*) AS COUNT FROM TEST GROUP BY LEN(DATA);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NEW_KEY` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.2.0_1608159557873/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.2.0_1608159557873/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_alias_(table-_table)/6.2.0_1608159557873/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_nulls/6.2.0_1608159559950/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_nulls/6.2.0_1608159559950/plan.json new file mode 100644 index 000000000000..35f2b9cad1ac --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_nulls/6.2.0_1608159559950/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n BAD_UDF(TEST.DATA) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY BAD_UDF(TEST.DATA)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "BAD_UDF(DATA)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_nulls/6.2.0_1608159559950/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_nulls/6.2.0_1608159559950/spec.json new file mode 100644 index 000000000000..6a8a63695807 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_nulls/6.2.0_1608159559950/spec.json @@ -0,0 +1,160 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159559950, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "single expression with nulls", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "data" : "d1" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_1" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "KSQL_COL_1" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT bad_udf(DATA), COUNT(*) FROM TEST GROUP BY bad_udf(DATA);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_nulls/6.2.0_1608159559950/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_nulls/6.2.0_1608159559950/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_single_expression_with_nulls/6.2.0_1608159559950/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_steam_with_no_key/6.2.0_1608159558011/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_steam_with_no_key/6.2.0_1608159558011/plan.json new file mode 100644 index 000000000000..e693b74ef3ef --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_steam_with_no_key/6.2.0_1608159558011/plan.json @@ -0,0 +1,180 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (DATA INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`DATA` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA DATA,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DATA` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`DATA` INTEGER" + }, + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "DATA" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_steam_with_no_key/6.2.0_1608159558011/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_steam_with_no_key/6.2.0_1608159558011/spec.json new file mode 100644 index 000000000000..dddc8da73b7a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_steam_with_no_key/6.2.0_1608159558011/spec.json @@ -0,0 +1,154 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159558011, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` INTEGER KEY, `DATA` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`DATA` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` INTEGER KEY, `DATA` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DATA` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "steam with no key", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : 22 + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : 333 + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "data" : 22 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 22, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 333, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 22, + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (data INT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) AS COUNT FROM TEST GROUP BY DATA;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DATA` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`DATA` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_steam_with_no_key/6.2.0_1608159558011/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_steam_with_no_key/6.2.0_1608159558011/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_steam_with_no_key/6.2.0_1608159558011/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(stream-_table)/6.2.0_1608159565021/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(stream-_table)/6.2.0_1608159565021/plan.json new file mode 100644 index 000000000000..6aaa59cbae39 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(stream-_table)/6.2.0_1608159565021/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, F1 STRING, F2 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F1` STRING, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n (TEST.F2 + TEST.F1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY (TEST.F2 + TEST.F1)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F1` STRING, `F2` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F2 AS F2", "F1 AS F1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "(F2 + F1)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F2", "F1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(stream-_table)/6.2.0_1608159565021/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(stream-_table)/6.2.0_1608159565021/spec.json new file mode 100644 index 000000000000..d73f1df9d408 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(stream-_table)/6.2.0_1608159565021/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159565021, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `F2` STRING, `F1` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `F2` STRING, `F1` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "string concat using + op (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "3,a" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "a1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "b2", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "a1", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "b2", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "a3", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, f1 VARCHAR, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f2 + f1, COUNT(*) FROM TEST GROUP BY f2 + f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(stream-_table)/6.2.0_1608159565021/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(stream-_table)/6.2.0_1608159565021/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(stream-_table)/6.2.0_1608159565021/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.2.0_1608159565114/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.2.0_1608159565114/plan.json new file mode 100644 index 000000000000..61e3ff073137 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.2.0_1608159565114/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, USER INTEGER, SUBREGION STRING, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `USER` INTEGER, `SUBREGION` STRING, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n (TEST.REGION + TEST.SUBREGION) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY (TEST.REGION + TEST.SUBREGION)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `USER` INTEGER, `SUBREGION` STRING, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "REGION AS REGION", "SUBREGION AS SUBREGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "(REGION + SUBREGION)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "SUBREGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.2.0_1608159565114/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.2.0_1608159565114/spec.json new file mode 100644 index 000000000000..55c2fd1634bc --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.2.0_1608159565114/spec.json @@ -0,0 +1,190 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159565114, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `REGION` STRING, `SUBREGION` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `USER` INTEGER, `SUBREGION` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `REGION` STRING, `SUBREGION` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "string concat using + op (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a,r0" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,a,r1" + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "3,a,r0" + }, { + "topic" : "test_topic", + "key" : "4", + "value" : "4,b,r0" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,a,r0" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,b,r1" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "r0a", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1a", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r0a", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r0b", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r0a", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1a", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "r0a", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r0a", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "r1b", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, user INT, subregion VARCHAR, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT region + subregion, COUNT(*) FROM TEST GROUP BY region + subregion;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` STRING KEY, `KSQL_COL_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `USER` INTEGER, `SUBREGION` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.2.0_1608159565114/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.2.0_1608159565114/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_string_concat_using_+_op_(table-_table)/6.2.0_1608159565114/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_field_(stream-_table)/6.2.0_1608159557277/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_field_(stream-_table)/6.2.0_1608159557277/plan.json new file mode 100644 index 000000000000..a88989660b43 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_field_(stream-_table)/6.2.0_1608159557277/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, ADDRESS STRUCT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.ADDRESS->TOWN TOWN,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.ADDRESS->TOWN\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`TOWN` STRING KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ADDRESS AS ADDRESS", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ADDRESS->TOWN" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ADDRESS", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "TOWN" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_field_(stream-_table)/6.2.0_1608159557277/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_field_(stream-_table)/6.2.0_1608159557277/spec.json new file mode 100644 index 000000000000..a20b5cb0e83f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_field_(stream-_table)/6.2.0_1608159557277/spec.json @@ -0,0 +1,193 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159557277, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`TOWN` STRING KEY, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`TOWN` STRING KEY, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`TOWN` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "struct field (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "ADDRESS" : { + "STREET" : "1st Steet", + "Town" : "Oxford" + } + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "ADDRESS" : { + "STREET" : "1st Steet", + "Town" : "London" + } + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "ADDRESS" : { + "STREET" : "1st Steet", + "Town" : "Oxford" + } + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "ADDRESS" : { + "STREET" : "1st Steet", + "Town" : "London" + } + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "ADDRESS" : { + "STREET" : "1st Steet", + "Town" : "Oxford" + } + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "Oxford", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "London", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "Oxford", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "London", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "Oxford", + "value" : { + "COUNT" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, ADDRESS STRUCT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT ADDRESS->TOWN, COUNT(*) AS COUNT FROM TEST GROUP BY ADDRESS->TOWN;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`TOWN` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `ADDRESS` STRUCT<`STREET` STRING, `TOWN` STRING>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_field_(stream-_table)/6.2.0_1608159557277/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_field_(stream-_table)/6.2.0_1608159557277/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_field_(stream-_table)/6.2.0_1608159557277/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558386/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558386/plan.json new file mode 100644 index 000000000000..30976e7492ee --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558386/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID INTEGER KEY, COL1 STRUCT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `COL1` STRUCT<`A` STRING, `B` INTEGER>", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.COL1->A NEW_KEY,\n AS_VALUE(INPUT.COL1->A) VV,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY INPUT.COL1->A\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NEW_KEY` STRING KEY, `VV` STRING, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `COL1` STRUCT<`A` STRING, `B` INTEGER>" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "COL1 AS COL1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "COL1->A" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "COL1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "NEW_KEY" ], + "selectExpressions" : [ "AS_VALUE(COL1->A) AS VV", "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558386/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558386/spec.json new file mode 100644 index 000000000000..e8c5f097b273 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558386/spec.json @@ -0,0 +1,188 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159558386, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`A` STRING KEY, `COL1` STRUCT<`A` STRING, `B` INTEGER>, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `COL1` STRUCT<`A` STRING, `B` INTEGER>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`A` STRING KEY, `COL1` STRUCT<`A` STRING, `B` INTEGER>, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NEW_KEY` STRING KEY, `VV` STRING, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "struct in group-by and non aggregate function in select", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "col1" : { + "a" : "lala", + "b" : 1 + } + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "col1" : { + "a" : "lala", + "b" : 2 + } + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "col1" : { + "a" : "wonderland", + "b" : 3 + } + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "col1" : { + "a" : "lamb", + "b" : 4 + } + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "col1" : null + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "lala", + "value" : { + "VV" : "lala", + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "lala", + "value" : { + "VV" : "lala", + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "wonderland", + "value" : { + "VV" : "wonderland", + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "lamb", + "value" : { + "VV" : "lamb", + "COUNT" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (id INT KEY, col1 STRUCT) WITH (kafka_topic='test_topic', value_format='json');", "CREATE TABLE OUTPUT AS SELECT col1->a AS NEW_KEY, AS_VALUE(col1->a) as VV, COUNT(*) AS COUNT FROM INPUT GROUP BY col1->a;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `COL1` STRUCT<`A` STRING, `B` INTEGER>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NEW_KEY` STRING KEY, `VV` STRING, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558386/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558386/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_struct_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558386/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_having/6.2.0_1608159558187/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_having/6.2.0_1608159558187/plan.json new file mode 100644 index 000000000000..33318ed46f08 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_having/6.2.0_1608159558187/plan.json @@ -0,0 +1,188 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID INTEGER KEY, COL1 MAP) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `COL1` MAP", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.COL1['foo'] NEW_KEY,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY INPUT.COL1['foo']\nHAVING (INPUT.COL1['foo'] = 'lala')\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `COL1` MAP" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "COL1 AS COL1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "COL1['foo']" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "COL1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "filterExpression" : "(COL1['foo'] = 'lala')" + }, + "keyColumnNames" : [ "NEW_KEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_having/6.2.0_1608159558187/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_having/6.2.0_1608159558187/spec.json new file mode 100644 index 000000000000..18a5e8f5d68e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_having/6.2.0_1608159558187/spec.json @@ -0,0 +1,162 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159558187, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL1` MAP, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `COL1` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL1` MAP, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "subscript in group-by and having", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "col1" : { + "foo" : "lala" + } + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "col1" : { + "foo" : "kaka" + } + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "col1" : { + "alice" : "wonderland" + } + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "col1" : { + "mary" : "lamb" + } + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "col1" : null + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "lala", + "value" : { + "COUNT" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (id INT KEY, col1 MAP) WITH (kafka_topic='test_topic', value_format='json');", "CREATE TABLE OUTPUT AS SELECT col1['foo'] AS NEW_KEY, COUNT(*) AS COUNT FROM INPUT GROUP BY col1['foo'] HAVING col1['foo']='lala';" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `COL1` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_having/6.2.0_1608159558187/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_having/6.2.0_1608159558187/topology new file mode 100644 index 000000000000..b08ea0de8518 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_having/6.2.0_1608159558187/topology @@ -0,0 +1,49 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000014 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000014 (stores: []) + --> KSTREAM-SINK-0000000015 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000015 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000014 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558296/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558296/plan.json new file mode 100644 index 000000000000..cbd3bf2daef1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558296/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID INTEGER KEY, COL1 MAP) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `COL1` MAP", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.COL1['foo'] NEW_KEY,\n AS_VALUE(INPUT.COL1['foo']) VV,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY INPUT.COL1['foo']\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NEW_KEY` STRING KEY, `VV` STRING, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `COL1` MAP" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "COL1 AS COL1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "COL1['foo']" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "COL1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "NEW_KEY" ], + "selectExpressions" : [ "AS_VALUE(COL1['foo']) AS VV", "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558296/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558296/spec.json new file mode 100644 index 000000000000..e8d505b46c27 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558296/spec.json @@ -0,0 +1,170 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159558296, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL1` MAP, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `COL1` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL1` MAP, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NEW_KEY` STRING KEY, `VV` STRING, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "subscript in group-by and non aggregate function in select", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "col1" : { + "foo" : "lala" + } + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "col1" : { + "foo" : "kaka" + } + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "col1" : { + "alice" : "wonderland" + } + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "col1" : { + "mary" : "lamb" + } + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "col1" : null + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "lala", + "value" : { + "VV" : "lala", + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "kaka", + "value" : { + "VV" : "kaka", + "COUNT" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (id INT KEY, col1 MAP) WITH (kafka_topic='test_topic', value_format='json');", "CREATE TABLE OUTPUT AS SELECT col1['foo'] AS NEW_KEY, AS_VALUE(col1['foo']) as VV, COUNT(*) AS COUNT FROM INPUT GROUP BY col1['foo'];" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `COL1` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NEW_KEY` STRING KEY, `VV` STRING, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558296/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558296/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_non_aggregate_function_in_select/6.2.0_1608159558296/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_select/6.2.0_1608159558099/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_select/6.2.0_1608159558099/plan.json new file mode 100644 index 000000000000..af1638f3cdb8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_select/6.2.0_1608159558099/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID INTEGER KEY, COL1 MAP) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `COL1` MAP", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.COL1['foo'] NEW_KEY,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY INPUT.COL1['foo']\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `COL1` MAP" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "COL1 AS COL1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "COL1['foo']" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "COL1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "NEW_KEY" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_select/6.2.0_1608159558099/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_select/6.2.0_1608159558099/spec.json new file mode 100644 index 000000000000..687ef08e6462 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_select/6.2.0_1608159558099/spec.json @@ -0,0 +1,168 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159558099, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL1` MAP, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `COL1` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `COL1` MAP, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "subscript in group-by and select", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "col1" : { + "foo" : "lala" + } + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "col1" : { + "foo" : "kaka" + } + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "col1" : { + "alice" : "wonderland" + } + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "col1" : { + "mary" : "lamb" + } + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "col1" : null + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "lala", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "kaka", + "value" : { + "COUNT" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (id INT KEY, col1 MAP) WITH (kafka_topic='test_topic', value_format='json');", "CREATE TABLE OUTPUT AS SELECT col1['foo'] AS NEW_KEY, COUNT(*) AS COUNT FROM INPUT GROUP BY col1['foo'];" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `COL1` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NEW_KEY` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_select/6.2.0_1608159558099/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_select/6.2.0_1608159558099/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_subscript_in_group-by_and_select/6.2.0_1608159558099/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_udafs_only_in_having_(stream-_table)/6.2.0_1608159557086/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_udafs_only_in_having_(stream-_table)/6.2.0_1608159557086/plan.json new file mode 100644 index 000000000000..5a1cbd73ad92 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_udafs_only_in_having_(stream-_table)/6.2.0_1608159557086/plan.json @@ -0,0 +1,187 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (NAME STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`NAME` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.NAME NAME,\n LEN(INPUT.NAME) LEN\nFROM INPUT INPUT\nGROUP BY INPUT.NAME\nHAVING (COUNT(INPUT.NAME) = 2)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NAME` STRING KEY, `LEN` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`NAME` STRING" + }, + "selectExpressions" : [ "NAME AS NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "NAME" ], + "aggregationFunctions" : [ "COUNT(NAME)" ] + }, + "filterExpression" : "(KSQL_AGG_VARIABLE_0 = 2)" + }, + "keyColumnNames" : [ "NAME" ], + "selectExpressions" : [ "LEN(NAME) AS LEN" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_udafs_only_in_having_(stream-_table)/6.2.0_1608159557086/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_udafs_only_in_having_(stream-_table)/6.2.0_1608159557086/spec.json new file mode 100644 index 000000000000..02a606a5e6d1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_udafs_only_in_having_(stream-_table)/6.2.0_1608159557086/spec.json @@ -0,0 +1,150 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159557086, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`NAME` STRING KEY, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`NAME` STRING KEY, `NAME` STRING, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NAME` STRING KEY, `LEN` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "udafs only in having (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "NAME" : "bob" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "NAME" : "bob" + }, + "timestamp" : 2 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "NAME" : "bob" + }, + "timestamp" : 3 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "bob", + "value" : { + "LEN" : 3 + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : "bob", + "value" : null + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (NAME STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT NAME, LEN(NAME) AS LEN FROM INPUT GROUP BY NAME HAVING COUNT(NAME) = 2;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NAME` STRING KEY, `LEN` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_udafs_only_in_having_(stream-_table)/6.2.0_1608159557086/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_udafs_only_in_having_(stream-_table)/6.2.0_1608159557086/topology new file mode 100644 index 000000000000..b08ea0de8518 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_udafs_only_in_having_(stream-_table)/6.2.0_1608159557086/topology @@ -0,0 +1,49 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000014 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000014 (stores: []) + --> KSTREAM-SINK-0000000015 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000015 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000014 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_value_column_(stream-_table)/6.2.0_1608159557175/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_value_column_(stream-_table)/6.2.0_1608159557175/plan.json new file mode 100644 index 000000000000..fc75451f4459 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_value_column_(stream-_table)/6.2.0_1608159557175/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA DATA,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DATA` STRING KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "DATA" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_value_column_(stream-_table)/6.2.0_1608159557175/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_value_column_(stream-_table)/6.2.0_1608159557175/spec.json new file mode 100644 index 000000000000..2d739541eae1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_value_column_(stream-_table)/6.2.0_1608159557175/spec.json @@ -0,0 +1,178 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159557175, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DATA` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "value column (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "data" : "d1" + } + }, { + "topic" : "test_topic", + "key" : 3, + "value" : { + "data" : "d2" + } + }, { + "topic" : "test_topic", + "key" : 4, + "value" : { + "data" : "d1" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : { + "COUNT" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : { + "COUNT" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) AS COUNT FROM TEST GROUP BY DATA;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DATA` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_value_column_(stream-_table)/6.2.0_1608159557175/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_value_column_(stream-_table)/6.2.0_1608159557175/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_value_column_(stream-_table)/6.2.0_1608159557175/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_windowed_aggregate_with_field_within_struct_key/6.2.0_1608159568328/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_windowed_aggregate_with_field_within_struct_key/6.2.0_1608159568328/plan.json new file mode 100644 index 000000000000..fe73c79400db --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_windowed_aggregate_with_field_within_struct_key/6.2.0_1608159568328/plan.json @@ -0,0 +1,192 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID STRUCT KEY, VAL INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` STRUCT<`F1` INTEGER, `F2` INTEGER> KEY, `VAL` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.ID->F1 F1,\n COUNT(1) COUNT\nFROM INPUT INPUT\nWINDOW TUMBLING ( SIZE 1 SECONDS ) \nGROUP BY INPUT.ID->F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F1` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamWindowedAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` STRUCT<`F1` INTEGER, `F2` INTEGER> KEY, `VAL` INTEGER" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "1 AS KSQL_INTERNAL_COL_1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "ID->F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "ID" ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_1)" ], + "windowExpression" : " TUMBLING ( SIZE 1 SECONDS ) " + }, + "keyColumnNames" : [ "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_windowed_aggregate_with_field_within_struct_key/6.2.0_1608159568328/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_windowed_aggregate_with_field_within_struct_key/6.2.0_1608159568328/spec.json new file mode 100644 index 000000000000..c57ed484bb15 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_windowed_aggregate_with_field_within_struct_key/6.2.0_1608159568328/spec.json @@ -0,0 +1,200 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159568328, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F1` INTEGER KEY, `ID` STRUCT<`F1` INTEGER, `F2` INTEGER>, `KSQL_INTERNAL_COL_1` INTEGER", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` STRUCT<`F1` INTEGER, `F2` INTEGER> KEY, `VAL` INTEGER", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F1` INTEGER KEY, `ID` STRUCT<`F1` INTEGER, `F2` INTEGER>, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F1` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "windowed aggregate with field within struct key", + "inputs" : [ { + "topic" : "test_topic", + "key" : { + "F1" : 1, + "F2" : 1 + }, + "value" : { + "VAL" : 0 + } + }, { + "topic" : "test_topic", + "key" : { + "F1" : 1, + "F2" : 1 + }, + "value" : { + "VAL" : 0 + } + }, { + "topic" : "test_topic", + "key" : { + "F1" : 1, + "F2" : 1 + }, + "value" : { + "VAL" : 0 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "COUNT" : 1 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "COUNT" : 2 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "COUNT" : 3 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (ID STRUCT KEY, VAL INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT as SELECT ID->F1, count(1) AS count FROM INPUT WINDOW TUMBLING (SIZE 1 SECOND) group by ID->F1;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` STRUCT<`F1` INTEGER, `F2` INTEGER> KEY, `VAL` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F1` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "windowType" : "TUMBLING", + "windowSize" : 1000 + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_windowed_aggregate_with_field_within_struct_key/6.2.0_1608159568328/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_windowed_aggregate_with_field_within_struct_key/6.2.0_1608159568328/topology new file mode 100644 index 000000000000..bdf02e6d2f71 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_windowed_aggregate_with_field_within_struct_key/6.2.0_1608159568328/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Aggregate-WindowSelect + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Aggregate-WindowSelect (stores: []) + --> Aggregate-Project + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- Aggregate-Aggregate-WindowSelect + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000013 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000012 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(stream-_table)/6.2.0_1608159563098/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(stream-_table)/6.2.0_1608159563098/plan.json new file mode 100644 index 000000000000..3eb9b2b9cbed --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(stream-_table)/6.2.0_1608159563098/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, DATA STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `DATA` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.DATA DATA,\n (COUNT(*) * 2) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.DATA\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `DATA` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "DATA AS DATA", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "DATA" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "DATA", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "DATA" ], + "selectExpressions" : [ "(KSQL_AGG_VARIABLE_0 * 2) AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(stream-_table)/6.2.0_1608159563098/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(stream-_table)/6.2.0_1608159563098/spec.json new file mode 100644 index 000000000000..3f5bb416902b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(stream-_table)/6.2.0_1608159563098/spec.json @@ -0,0 +1,142 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159563098, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`DATA` STRING KEY, `DATA` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "with aggregate arithmetic (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "d1" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "d2" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "d1" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "d1", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "d2", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "d1", + "value" : "4" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*)*2 FROM TEST GROUP BY DATA;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DATA` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `DATA` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(stream-_table)/6.2.0_1608159563098/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(stream-_table)/6.2.0_1608159563098/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(stream-_table)/6.2.0_1608159563098/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.2.0_1608159563227/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.2.0_1608159563227/plan.json new file mode 100644 index 000000000000..09f43ad47151 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.2.0_1608159563227/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, USER INTEGER, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.REGION REGION,\n (COUNT(*) * 2) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "REGION AS REGION", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "(KSQL_AGG_VARIABLE_0 * 2) AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.2.0_1608159563227/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.2.0_1608159563227/spec.json new file mode 100644 index 000000000000..1c324c94ad5f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.2.0_1608159563227/spec.json @@ -0,0 +1,170 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159563227, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "with aggregate arithmetic (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1,r0" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r1" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3,r0" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : null + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,r0" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "4" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "r1", + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : "r0", + "value" : "4" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT region, COUNT(*) * 2 FROM TEST GROUP BY region;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `USER` INTEGER, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.2.0_1608159563227/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.2.0_1608159563227/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_(table-_table)/6.2.0_1608159563227/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(stream-_table)/6.2.0_1608159563438/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(stream-_table)/6.2.0_1608159563438/plan.json new file mode 100644 index 000000000000..bb118e3454ff --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(stream-_table)/6.2.0_1608159563438/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, ITEM INTEGER, COST INTEGER) WITH (FORMAT='DELIMITED', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ITEM` INTEGER, `COST` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.ITEM ITEM,\n TEST.COST COST,\n (TEST.COST * COUNT(*)) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.ITEM, TEST.COST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ITEM` INTEGER KEY, `COST` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `ITEM` INTEGER, `COST` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ITEM AS ITEM", "COST AS COST", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "ITEM", "COST" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "ITEM", "COST", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "ITEM", "COST" ], + "selectExpressions" : [ "(COST * KSQL_AGG_VARIABLE_0) AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(stream-_table)/6.2.0_1608159563438/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(stream-_table)/6.2.0_1608159563438/spec.json new file mode 100644 index 000000000000..92c76c9aa999 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(stream-_table)/6.2.0_1608159563438/spec.json @@ -0,0 +1,150 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159563438, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`ITEM` INTEGER KEY, `COST` INTEGER KEY, `ITEM` INTEGER, `COST` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ITEM` INTEGER, `COST` INTEGER", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ITEM` INTEGER KEY, `COST` INTEGER KEY, `ITEM` INTEGER, `COST` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ITEM` INTEGER KEY, `COST` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "with aggregate arithmetic involving source field (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "1,10" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "1,20" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "2,30" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "1,10" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1,10", + "value" : "10" + }, { + "topic" : "OUTPUT", + "key" : "1,20", + "value" : "20" + }, { + "topic" : "OUTPUT", + "key" : "2,30", + "value" : "30" + }, { + "topic" : "OUTPUT", + "key" : "1,10", + "value" : "20" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, ITEM INT, COST INT) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT ITEM, COST, COST * COUNT() FROM TEST GROUP BY ITEM, COST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`ITEM` INTEGER KEY, `COST` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ITEM` INTEGER, `COST` INTEGER", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(stream-_table)/6.2.0_1608159563438/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(stream-_table)/6.2.0_1608159563438/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(stream-_table)/6.2.0_1608159563438/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.2.0_1608159563561/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.2.0_1608159563561/plan.json new file mode 100644 index 000000000000..83934eef8c29 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.2.0_1608159563561/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, F0 INTEGER, F1 INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F0` INTEGER, `F1` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F0 F0,\n (TEST.F0 * SUM(TEST.F1)) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F0\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F0` INTEGER KEY, `KSQL_COL_0` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F0` INTEGER, `F1` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F0 AS F0", "F1 AS F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F0" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F0", "F1" ], + "aggregationFunctions" : [ "SUM(F1)" ] + }, + "keyColumnNames" : [ "F0" ], + "selectExpressions" : [ "(F0 * KSQL_AGG_VARIABLE_0) AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.2.0_1608159563561/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.2.0_1608159563561/spec.json new file mode 100644 index 000000000000..401b25a70f4f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.2.0_1608159563561/spec.json @@ -0,0 +1,166 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159563561, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F0` INTEGER KEY, `F0` INTEGER, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F0` INTEGER, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F0` INTEGER KEY, `F0` INTEGER, `F1` INTEGER, `KSQL_AGG_VARIABLE_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F0` INTEGER KEY, `KSQL_COL_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "with aggregate arithmetic involving source field (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 2, + "value" : "2,10" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,20" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2,30" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 2, + "value" : "20" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "40" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "0" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "60" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "0" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID INT PRIMARY KEY, f0 INT, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f0, f0 * SUM(f1) FROM TEST GROUP BY f0;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F0` INTEGER KEY, `KSQL_COL_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `F0` INTEGER, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.2.0_1608159563561/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.2.0_1608159563561/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_aggregate_arithmetic_involving_source_field_(table-_table)/6.2.0_1608159563561/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constant_having_(stream-table)/6.2.0_1608159565874/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constant_having_(stream-table)/6.2.0_1608159565874/plan.json new file mode 100644 index 000000000000..4c49fce9b2d6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constant_having_(stream-table)/6.2.0_1608159565874/plan.json @@ -0,0 +1,188 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F2 F2,\n SUM(TEST.F1) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2\nHAVING (TEST.F2 = 'test')\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `KSQL_COL_0` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F2 AS F2", "F1 AS F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F2", "F1" ], + "aggregationFunctions" : [ "SUM(F1)" ] + }, + "filterExpression" : "(F2 = 'test')" + }, + "keyColumnNames" : [ "F2" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constant_having_(stream-table)/6.2.0_1608159565874/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constant_having_(stream-table)/6.2.0_1608159565874/spec.json new file mode 100644 index 000000000000..6ee8e4996030 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constant_having_(stream-table)/6.2.0_1608159565874/spec.json @@ -0,0 +1,146 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159565874, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F2` STRING, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F2` STRING, `F1` INTEGER, `KSQL_AGG_VARIABLE_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `KSQL_COL_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "with constant having (stream-table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : "0", + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : "0", + "value" : "2,test" + }, { + "topic" : "test_topic", + "key" : "0", + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : "0", + "value" : "3,test" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "test", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "test", + "value" : "5" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f2, SUM(f1) FROM TEST GROUP BY f2 HAVING f2='test';" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `KSQL_COL_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constant_having_(stream-table)/6.2.0_1608159565874/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constant_having_(stream-table)/6.2.0_1608159565874/topology new file mode 100644 index 000000000000..b08ea0de8518 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constant_having_(stream-table)/6.2.0_1608159565874/topology @@ -0,0 +1,49 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000014 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000014 (stores: []) + --> KSTREAM-SINK-0000000015 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000015 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000014 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constants_in_the_projection_(stream-_table)/6.2.0_1608159565969/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constants_in_the_projection_(stream-_table)/6.2.0_1608159565969/plan.json new file mode 100644 index 000000000000..a5050a7b6bce --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constants_in_the_projection_(stream-_table)/6.2.0_1608159565969/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, F1 INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n 'some constant' F3,\n COUNT(TEST.F1) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F1` INTEGER KEY, `F3` STRING, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1" ], + "aggregationFunctions" : [ "COUNT(F1)" ] + }, + "keyColumnNames" : [ "F1" ], + "selectExpressions" : [ "'some constant' AS F3", "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constants_in_the_projection_(stream-_table)/6.2.0_1608159565969/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constants_in_the_projection_(stream-_table)/6.2.0_1608159565969/spec.json new file mode 100644 index 000000000000..07526001ef12 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constants_in_the_projection_(stream-_table)/6.2.0_1608159565969/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159565969, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F1` INTEGER KEY, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F1` INTEGER KEY, `F1` INTEGER, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F1` INTEGER KEY, `F3` STRING, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "with constants in the projection (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : "1" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : "some constant,1" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "some constant,1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "some constant,2" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "some constant,2" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "some constant,1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, 'some constant' as f3, COUNT(f1) FROM TEST GROUP BY f1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F1` INTEGER KEY, `F3` STRING, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constants_in_the_projection_(stream-_table)/6.2.0_1608159565969/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constants_in_the_projection_(stream-_table)/6.2.0_1608159565969/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_constants_in_the_projection_(stream-_table)/6.2.0_1608159565969/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(stream-_table)/6.2.0_1608159565474/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(stream-_table)/6.2.0_1608159565474/plan.json new file mode 100644 index 000000000000..577a7bc2d5a3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(stream-_table)/6.2.0_1608159565474/plan.json @@ -0,0 +1,188 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, F1 INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F1\nHAVING (SUM(TEST.F1) > 1)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)", "SUM(F1)" ] + }, + "filterExpression" : "(KSQL_AGG_VARIABLE_1 > 1)" + }, + "keyColumnNames" : [ "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(stream-_table)/6.2.0_1608159565474/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(stream-_table)/6.2.0_1608159565474/spec.json new file mode 100644 index 000000000000..1ceeb1a2f552 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(stream-_table)/6.2.0_1608159565474/spec.json @@ -0,0 +1,154 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159565474, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F1` INTEGER KEY, `F1` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F1` INTEGER KEY, `F1` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT, `KSQL_AGG_VARIABLE_1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "with having expression (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 1, + "value" : "1" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : "1" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "2" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "3" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 2, + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 2, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 3, + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, COUNT(*) FROM TEST GROUP BY f1 HAVING SUM(f1) > 1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(stream-_table)/6.2.0_1608159565474/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(stream-_table)/6.2.0_1608159565474/topology new file mode 100644 index 000000000000..b08ea0de8518 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(stream-_table)/6.2.0_1608159565474/topology @@ -0,0 +1,49 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000014 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000014 (stores: []) + --> KSTREAM-SINK-0000000015 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000015 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000014 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.2.0_1608159565561/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.2.0_1608159565561/plan.json new file mode 100644 index 000000000000..7cae3d581a54 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.2.0_1608159565561/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, F0 INTEGER, F1 INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n SUM(TEST.F0) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F1\nHAVING (COUNT(TEST.F1) > 0)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F1` INTEGER KEY, `KSQL_COL_0` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F1 AS F1", "F0 AS F0" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1", "F0" ], + "aggregationFunctions" : [ "SUM(F0)", "COUNT(F1)" ] + }, + "filterExpression" : "(KSQL_AGG_VARIABLE_1 > 0)" + }, + "keyColumnNames" : [ "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.2.0_1608159565561/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.2.0_1608159565561/spec.json new file mode 100644 index 000000000000..ea56c8b42446 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.2.0_1608159565561/spec.json @@ -0,0 +1,181 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159565561, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F1` INTEGER KEY, `F1` INTEGER, `F0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F1` INTEGER KEY, `F1` INTEGER, `F0` INTEGER, `KSQL_AGG_VARIABLE_0` INTEGER, `KSQL_AGG_VARIABLE_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F1` INTEGER KEY, `KSQL_COL_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "with having expression (table->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,0", + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,1", + "timestamp" : 2 + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null, + "timestamp" : 3 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "3,0", + "timestamp" : 4 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,0", + "timestamp" : 5 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : "1", + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2", + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null, + "timestamp" : 3 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : "3", + "timestamp" : 4 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 5 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : "5", + "timestamp" : 5 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, f0 INT, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, SUM(f0) FROM TEST GROUP BY f1 HAVING COUNT(f1) > 0;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F1` INTEGER KEY, `KSQL_COL_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `F0` INTEGER, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.2.0_1608159565561/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.2.0_1608159565561/topology new file mode 100644 index 000000000000..3a244cb18785 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_(table-_table)/6.2.0_1608159565561/topology @@ -0,0 +1,52 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000016 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_on_non-group-by_field_(stream-_table)/6.2.0_1608159565783/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_on_non-group-by_field_(stream-_table)/6.2.0_1608159565783/plan.json new file mode 100644 index 000000000000..1adf1a61636f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_on_non-group-by_field_(stream-_table)/6.2.0_1608159565783/plan.json @@ -0,0 +1,188 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F2 F2,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F2\nHAVING (SUM(TEST.F1) > 10)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F2` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F2 AS F2", "ROWTIME AS ROWTIME", "F1 AS F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F2", "ROWTIME", "F1" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)", "SUM(F1)" ] + }, + "filterExpression" : "(KSQL_AGG_VARIABLE_1 > 10)" + }, + "keyColumnNames" : [ "F2" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_on_non-group-by_field_(stream-_table)/6.2.0_1608159565783/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_on_non-group-by_field_(stream-_table)/6.2.0_1608159565783/spec.json new file mode 100644 index 000000000000..16dad6932044 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_on_non-group-by_field_(stream-_table)/6.2.0_1608159565783/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159565783, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F2` STRING KEY, `F2` STRING, `ROWTIME` BIGINT, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F2` STRING KEY, `F2` STRING, `ROWTIME` BIGINT, `F1` INTEGER, `KSQL_AGG_VARIABLE_0` BIGINT, `KSQL_AGG_VARIABLE_1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F2` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "with having expression on non-group-by field (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "-", + "value" : "5,a" + }, { + "topic" : "test_topic", + "key" : "-", + "value" : "10,b" + }, { + "topic" : "test_topic", + "key" : "-", + "value" : "6,a" + }, { + "topic" : "test_topic", + "key" : "-", + "value" : "1,b" + }, { + "topic" : "test_topic", + "key" : "-", + "value" : "-1,a" + }, { + "topic" : "test_topic", + "key" : "-", + "value" : "1,a" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "a", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "b", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "a", + "value" : null + }, { + "topic" : "OUTPUT", + "key" : "a", + "value" : "4" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f2, COUNT(*) FROM TEST GROUP BY f2 HAVING SUM(f1) > 10;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F2` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_on_non-group-by_field_(stream-_table)/6.2.0_1608159565783/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_on_non-group-by_field_(stream-_table)/6.2.0_1608159565783/topology new file mode 100644 index 000000000000..b08ea0de8518 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_having_expression_on_non-group-by_field_(stream-_table)/6.2.0_1608159565783/topology @@ -0,0 +1,49 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000014 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000014 (stores: []) + --> KSTREAM-SINK-0000000015 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000015 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000014 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_multiple_having_expressions_(stream-_table)/6.2.0_1608159565677/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_multiple_having_expressions_(stream-_table)/6.2.0_1608159565677/plan.json new file mode 100644 index 000000000000..68fe6655d7c1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_multiple_having_expressions_(stream-_table)/6.2.0_1608159565677/plan.json @@ -0,0 +1,188 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, F1 INTEGER, F2 STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n COUNT(TEST.F1) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F1\nHAVING ((COUNT(TEST.F1) > 1) AND (TEST.F1 = 1))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableFilterV1", + "properties" : { + "queryContext" : "Aggregate/HavingFilter" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "F1 AS F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1" ], + "aggregationFunctions" : [ "COUNT(F1)", "COUNT(F1)" ] + }, + "filterExpression" : "((KSQL_AGG_VARIABLE_1 > 1) AND (F1 = 1))" + }, + "keyColumnNames" : [ "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_multiple_having_expressions_(stream-_table)/6.2.0_1608159565677/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_multiple_having_expressions_(stream-_table)/6.2.0_1608159565677/spec.json new file mode 100644 index 000000000000..196a30b7269a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_multiple_having_expressions_(stream-_table)/6.2.0_1608159565677/spec.json @@ -0,0 +1,150 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159565677, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F1` INTEGER KEY, `F1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F1` INTEGER KEY, `F1` INTEGER, `KSQL_AGG_VARIABLE_0` BIGINT, `KSQL_AGG_VARIABLE_1` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "with multiple having expressions (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : "1,a" + }, { + "topic" : "test_topic", + "key" : "0", + "value" : "2,b" + }, { + "topic" : "test_topic", + "key" : "0", + "value" : "1,test" + }, { + "topic" : "test_topic", + "key" : "0", + "value" : "2,test" + }, { + "topic" : "test_topic", + "key" : "0", + "value" : "2,test" + }, { + "topic" : "test_topic", + "key" : "0", + "value" : "1,test" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : "3" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, COUNT(f1) FROM TEST GROUP BY f1 HAVING COUNT(f1) > 1 AND f1=1;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `F1` INTEGER, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_multiple_having_expressions_(stream-_table)/6.2.0_1608159565677/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_multiple_having_expressions_(stream-_table)/6.2.0_1608159565677/topology new file mode 100644 index 000000000000..b08ea0de8518 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_multiple_having_expressions_(stream-_table)/6.2.0_1608159565677/topology @@ -0,0 +1,49 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-HavingFilter-ApplyPredicate + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-HavingFilter-ApplyPredicate (stores: []) + --> Aggregate-HavingFilter-Filter + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-HavingFilter-Filter (stores: []) + --> Aggregate-HavingFilter-PostProcess + <-- Aggregate-HavingFilter-ApplyPredicate + Processor: Aggregate-HavingFilter-PostProcess (stores: []) + --> Aggregate-Project + <-- Aggregate-HavingFilter-Filter + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000014 + <-- Aggregate-HavingFilter-PostProcess + Processor: KTABLE-TOSTREAM-0000000014 (stores: []) + --> KSTREAM-SINK-0000000015 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000015 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000014 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_single_grouping_set_(stream-_table)/6.2.0_1608159561261/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_single_grouping_set_(stream-_table)/6.2.0_1608159561261/plan.json new file mode 100644 index 000000000000..cc275fc07078 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_single_grouping_set_(stream-_table)/6.2.0_1608159561261/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, F1 INTEGER, F2 STRING, F3 INTEGER) WITH (FORMAT='DELIMITED', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING, `F3` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.F1 F1,\n TEST.F2 F2,\n TEST.F3 F3,\n COUNT(*) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.F3, TEST.F2, TEST.F1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`F3` INTEGER KEY, `F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING, `F3` INTEGER" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "F1 AS F1", "F2 AS F2", "F3 AS F3", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "F3", "F2", "F1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "F1", "F2", "F3", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "F3", "F2", "F1" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_single_grouping_set_(stream-_table)/6.2.0_1608159561261/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_single_grouping_set_(stream-_table)/6.2.0_1608159561261/spec.json new file mode 100644 index 000000000000..ba959ce6ab32 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_single_grouping_set_(stream-_table)/6.2.0_1608159561261/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159561261, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`F3` INTEGER KEY, `F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `F3` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING, `F3` INTEGER", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`F3` INTEGER KEY, `F2` STRING KEY, `F1` INTEGER KEY, `F1` INTEGER, `F2` STRING, `F3` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`F3` INTEGER KEY, `F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "with single grouping set (stream->table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a,-1" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,b,-2" + }, { + "topic" : "test_topic", + "key" : "1", + "value" : "1,a,-1" + }, { + "topic" : "test_topic", + "key" : "2", + "value" : "2,b,-2" + }, { + "topic" : "test_topic", + "key" : "3", + "value" : "3,a,-3" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "\"-1\",a,1", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "\"-2\",b,2", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "\"-1\",a,1", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "\"-2\",b,2", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "\"-3\",a,3", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR, f3 INT) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, f2, f3, COUNT(*) FROM TEST GROUP BY (f3, f2, f1);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`F3` INTEGER KEY, `F2` STRING KEY, `F1` INTEGER KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `F1` INTEGER, `F2` STRING, `F3` INTEGER", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_single_grouping_set_(stream-_table)/6.2.0_1608159561261/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_single_grouping_set_(stream-_table)/6.2.0_1608159561261/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_with_single_grouping_set_(stream-_table)/6.2.0_1608159561261/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(stream)/6.2.0_1608159567805/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(stream)/6.2.0_1608159567805/plan.json new file mode 100644 index 000000000000..59f2d449264d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(stream)/6.2.0_1608159567805/plan.json @@ -0,0 +1,180 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n 1 K,\n COUNT(1) ID\nFROM INPUT INPUT\nGROUP BY 1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `ID` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`VALUE` INTEGER" + }, + "selectExpressions" : [ "1 AS KSQL_INTERNAL_COL_0" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_0)" ] + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS ID" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(stream)/6.2.0_1608159567805/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(stream)/6.2.0_1608159567805/spec.json new file mode 100644 index 000000000000..b115e883f095 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(stream)/6.2.0_1608159567805/spec.json @@ -0,0 +1,154 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159567805, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_INTERNAL_COL_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "zero non-agg columns (stream)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "VALUE" : 0 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (VALUE INT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT 1 as k, count(1) AS ID FROM INPUT group by 1;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(stream)/6.2.0_1608159567805/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(stream)/6.2.0_1608159567805/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(stream)/6.2.0_1608159567805/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.2.0_1608159567951/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.2.0_1608159567951/plan.json new file mode 100644 index 000000000000..2ef027b55ee8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.2.0_1608159567951/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (ID INTEGER PRIMARY KEY, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`ID` INTEGER KEY, `VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n 1 K,\n COUNT(1) ID\nFROM INPUT INPUT\nGROUP BY 1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `ID` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` INTEGER KEY, `VALUE` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "1 AS KSQL_INTERNAL_COL_0" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_0)" ] + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS ID" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.2.0_1608159567951/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.2.0_1608159567951/spec.json new file mode 100644 index 000000000000..bd2350d9f1d1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.2.0_1608159567951/spec.json @@ -0,0 +1,162 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159567951, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_INTERNAL_COL_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "zero non-agg columns (table)", + "inputs" : [ { + "topic" : "test_topic", + "key" : 10, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 1666, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 98, + "value" : { + "VALUE" : 0 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 2 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 3 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (ID INT PRIMARY KEY, VALUE INT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT 1 as k, count(1) AS ID FROM INPUT group by 1;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.2.0_1608159567951/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.2.0_1608159567951/topology new file mode 100644 index 000000000000..b0fa6df1f29c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(table)/6.2.0_1608159567951/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(windowed_stream)/6.2.0_1608159567873/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(windowed_stream)/6.2.0_1608159567873/plan.json new file mode 100644 index 000000000000..a0e6b4331a15 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(windowed_stream)/6.2.0_1608159567873/plan.json @@ -0,0 +1,185 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`VALUE` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n 1 K,\n COUNT(1) ID\nFROM INPUT INPUT\nWINDOW TUMBLING ( SIZE 1 SECONDS ) \nGROUP BY 1\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `ID` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamWindowedAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`VALUE` INTEGER" + }, + "selectExpressions" : [ "1 AS KSQL_INTERNAL_COL_0" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_0)" ], + "windowExpression" : " TUMBLING ( SIZE 1 SECONDS ) " + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS ID" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(windowed_stream)/6.2.0_1608159567873/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(windowed_stream)/6.2.0_1608159567873/spec.json new file mode 100644 index 000000000000..5770d84be410 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(windowed_stream)/6.2.0_1608159567873/spec.json @@ -0,0 +1,183 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159567873, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_INTERNAL_COL_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` INTEGER KEY, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA", + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "zero non-agg columns (windowed stream)", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "VALUE" : 0 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 1 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 2 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "ID" : 3 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (VALUE INT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT 1 as k, count(1) AS ID FROM INPUT WINDOW TUMBLING (SIZE 1 SECOND) group by 1;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA", + "windowType" : "TUMBLING", + "windowSize" : 1000 + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA", + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA", + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(windowed_stream)/6.2.0_1608159567873/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(windowed_stream)/6.2.0_1608159567873/topology new file mode 100644 index 000000000000..bdf02e6d2f71 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_zero_non-agg_columns_(windowed_stream)/6.2.0_1608159567873/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Aggregate-WindowSelect + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Aggregate-WindowSelect (stores: []) + --> Aggregate-Project + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- Aggregate-Aggregate-WindowSelect + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000013 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000012 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.2.0_1608159569476/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.2.0_1608159569476/plan.json new file mode 100644 index 000000000000..0719aa225a09 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.2.0_1608159569476/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, ID BIGINT, NAME STRING, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE COUNT_BY_REGION AS SELECT\n TEST.REGION REGION,\n HISTOGRAM(TEST.NAME) COUNTS\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "COUNT_BY_REGION", + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "topicName" : "COUNT_BY_REGION", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "COUNT_BY_REGION", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "COUNT_BY_REGION" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "REGION AS REGION", "NAME AS NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "REGION", "NAME" ], + "aggregationFunctions" : [ "HISTOGRAM(NAME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNTS" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "COUNT_BY_REGION" + }, + "queryId" : "CTAS_COUNT_BY_REGION_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.2.0_1608159569476/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.2.0_1608159569476/spec.json new file mode 100644 index 000000000000..162058d90884 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.2.0_1608159569476/spec.json @@ -0,0 +1,362 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159569476, + "path" : "query-validation-tests/histogram.json", + "schemas" : { + "CTAS_COUNT_BY_REGION_0.Aggregate.GroupBy" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_COUNT_BY_REGION_0.COUNT_BY_REGION" : { + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_COUNT_BY_REGION_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `NAME` STRING, `KSQL_AGG_VARIABLE_0` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_COUNT_BY_REGION_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "histogram on a table - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : { + "ID" : 0, + "NAME" : "alice", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "ID" : 2, + "NAME" : "carol", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "ID" : 3, + "NAME" : "dave", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + } ], + "outputs" : [ { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "REGION", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "COUNT_BY_REGION", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, ID bigint, NAME varchar, REGION string) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE TABLE COUNT_BY_REGION AS SELECT region, histogram(name) AS COUNTS FROM TEST GROUP BY region;" ], + "post" : { + "sources" : [ { + "name" : "COUNT_BY_REGION", + "type" : "TABLE", + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "REGION", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + } ] + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "REGION", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "COUNT_BY_REGION", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "COUNTS", + "type" : [ "null", { + "type" : "array", + "items" : { + "type" : "record", + "name" : "KsqlDataSourceSchema_COUNTS", + "fields" : [ { + "name" : "key", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "value", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.internal.type" : "MapEntry" + } + } ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "REGION", + "type" : [ "null", "string" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "REGION", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", { + "type" : "array", + "items" : { + "type" : "record", + "name" : "KsqlDataSourceSchema_KSQL_AGG_VARIABLE_0", + "fields" : [ { + "name" : "key", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "value", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.internal.type" : "MapEntry" + } + } ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.2.0_1608159569476/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.2.0_1608159569476/topology new file mode 100644 index 000000000000..7441e4b0dc61 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_AVRO/6.2.0_1608159569476/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: COUNT_BY_REGION) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.2.0_1608159569640/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.2.0_1608159569640/plan.json new file mode 100644 index 000000000000..ab7b420287a2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.2.0_1608159569640/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, ID BIGINT, NAME STRING, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE COUNT_BY_REGION AS SELECT\n TEST.REGION REGION,\n HISTOGRAM(TEST.NAME) COUNTS\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "COUNT_BY_REGION", + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "topicName" : "COUNT_BY_REGION", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "COUNT_BY_REGION", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "COUNT_BY_REGION" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "REGION AS REGION", "NAME AS NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "REGION", "NAME" ], + "aggregationFunctions" : [ "HISTOGRAM(NAME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNTS" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "COUNT_BY_REGION" + }, + "queryId" : "CTAS_COUNT_BY_REGION_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.2.0_1608159569640/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.2.0_1608159569640/spec.json new file mode 100644 index 000000000000..cd5d6bd68977 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.2.0_1608159569640/spec.json @@ -0,0 +1,231 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159569640, + "path" : "query-validation-tests/histogram.json", + "schemas" : { + "CTAS_COUNT_BY_REGION_0.Aggregate.GroupBy" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_COUNT_BY_REGION_0.COUNT_BY_REGION" : { + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_COUNT_BY_REGION_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `NAME` STRING, `KSQL_AGG_VARIABLE_0` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_COUNT_BY_REGION_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "histogram on a table - JSON", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : { + "ID" : 0, + "NAME" : "alice", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "ID" : 2, + "NAME" : "carol", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "ID" : 3, + "NAME" : "dave", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + } ], + "outputs" : [ { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "COUNT_BY_REGION", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, ID bigint, NAME varchar, REGION string) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE COUNT_BY_REGION AS SELECT region, histogram(name) AS COUNTS FROM TEST GROUP BY region;" ], + "post" : { + "sources" : [ { + "name" : "COUNT_BY_REGION", + "type" : "TABLE", + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "COUNT_BY_REGION", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.2.0_1608159569640/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.2.0_1608159569640/topology new file mode 100644 index 000000000000..7441e4b0dc61 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_JSON/6.2.0_1608159569640/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: COUNT_BY_REGION) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.2.0_1608159569776/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.2.0_1608159569776/plan.json new file mode 100644 index 000000000000..aea514692fce --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.2.0_1608159569776/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, ID BIGINT, NAME STRING, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE COUNT_BY_REGION AS SELECT\n TEST.REGION REGION,\n HISTOGRAM(TEST.NAME) COUNTS\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "COUNT_BY_REGION", + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "topicName" : "COUNT_BY_REGION", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "COUNT_BY_REGION", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "COUNT_BY_REGION" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "REGION AS REGION", "NAME AS NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "nonAggregateColumns" : [ "REGION", "NAME" ], + "aggregationFunctions" : [ "HISTOGRAM(NAME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNTS" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "topicName" : "COUNT_BY_REGION" + }, + "queryId" : "CTAS_COUNT_BY_REGION_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.2.0_1608159569776/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.2.0_1608159569776/spec.json new file mode 100644 index 000000000000..af0724a1c7ff --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.2.0_1608159569776/spec.json @@ -0,0 +1,238 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159569776, + "path" : "query-validation-tests/histogram.json", + "schemas" : { + "CTAS_COUNT_BY_REGION_0.Aggregate.GroupBy" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_COUNT_BY_REGION_0.COUNT_BY_REGION" : { + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_COUNT_BY_REGION_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `NAME` STRING, `KSQL_AGG_VARIABLE_0` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + }, + "CTAS_COUNT_BY_REGION_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + } + } + }, + "testCase" : { + "name" : "histogram on a table - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : { + "ID" : 0, + "NAME" : "alice", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "ID" : 2, + "NAME" : "carol", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "ID" : 3, + "NAME" : "dave", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + } ], + "outputs" : [ { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n string NAME = 2;\n string REGION = 3;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "COUNT_BY_REGION", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, ID bigint, NAME varchar, REGION string) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE COUNT_BY_REGION AS SELECT region, histogram(name) AS COUNTS FROM TEST GROUP BY region;" ], + "post" : { + "sources" : [ { + "name" : "COUNT_BY_REGION", + "type" : "TABLE", + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string REGION = 1;\n string NAME = 2;\n}\n" + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n string NAME = 2;\n string REGION = 3;\n}\n" + }, { + "name" : "COUNT_BY_REGION", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n repeated ConnectDefault2Entry COUNTS = 1;\n\n message ConnectDefault2Entry {\n string key = 1;\n int64 value = 2;\n }\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 ID = 1;\n string NAME = 2;\n string REGION = 3;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF" + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string REGION = 1;\n string NAME = 2;\n repeated ConnectDefault2Entry KSQL_AGG_VARIABLE_0 = 3;\n\n message ConnectDefault2Entry {\n string key = 1;\n int64 value = 2;\n }\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.2.0_1608159569776/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.2.0_1608159569776/topology new file mode 100644 index 000000000000..7441e4b0dc61 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF/6.2.0_1608159569776/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: COUNT_BY_REGION) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_count/6.2.0_1608159570343/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_count/6.2.0_1608159570343/plan.json new file mode 100644 index 000000000000..0d9f35c9503e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_count/6.2.0_1608159570343/plan.json @@ -0,0 +1,186 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ROWKEY INTEGER KEY, ID INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ROWKEY` INTEGER KEY, `ID` INTEGER", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COUNT(1) COUNT\nFROM TEST TEST\nWINDOW HOPPING ( SIZE 5 SECONDS , ADVANCE BY 1 SECONDS ) \nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamWindowedAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `ID` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "ID AS ID", "1 AS KSQL_INTERNAL_COL_1" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID" ], + "aggregationFunctions" : [ "COUNT(KSQL_INTERNAL_COL_1)" ], + "windowExpression" : " HOPPING ( SIZE 5 SECONDS , ADVANCE BY 1 SECONDS ) " + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "S2" + }, + "queryId" : "CTAS_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_count/6.2.0_1608159570343/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_count/6.2.0_1608159570343/spec.json new file mode 100644 index 000000000000..4534147eeea7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_count/6.2.0_1608159570343/spec.json @@ -0,0 +1,266 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159570343, + "path" : "query-validation-tests/hopping-windows.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` INTEGER KEY, `ID` INTEGER, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `ID` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA", + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` INTEGER KEY, `ID` INTEGER, `KSQL_INTERNAL_COL_1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "count", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "id" : 0 + }, + "timestamp" : 10345 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "id" : 0 + }, + "timestamp" : 13251 + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 10345, + "window" : { + "start" : 6000, + "end" : 11000, + "type" : "TIME" + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 10345, + "window" : { + "start" : 7000, + "end" : 12000, + "type" : "TIME" + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 10345, + "window" : { + "start" : 8000, + "end" : 13000, + "type" : "TIME" + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 10345, + "window" : { + "start" : 9000, + "end" : 14000, + "type" : "TIME" + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 10345, + "window" : { + "start" : 10000, + "end" : 15000, + "type" : "TIME" + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNT" : 2 + }, + "timestamp" : 13251, + "window" : { + "start" : 9000, + "end" : 14000, + "type" : "TIME" + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNT" : 2 + }, + "timestamp" : 13251, + "window" : { + "start" : 10000, + "end" : 15000, + "type" : "TIME" + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 13251, + "window" : { + "start" : 11000, + "end" : 16000, + "type" : "TIME" + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 13251, + "window" : { + "start" : 12000, + "end" : 17000, + "type" : "TIME" + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 13251, + "window" : { + "start" : 13000, + "end" : 18000, + "type" : "TIME" + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ROWKEY INT KEY, ID INT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE S2 as SELECT ID, count(1) as count FROM test WINDOW HOPPING (SIZE 5 SECOND, ADVANCE BY 1 SECONDS) group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA", + "windowType" : "HOPPING", + "windowSize" : 5000 + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `ID` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA", + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA", + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_count/6.2.0_1608159570343/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_count/6.2.0_1608159570343/topology new file mode 100644 index 000000000000..d2b377a3888c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_count/6.2.0_1608159570343/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Aggregate-WindowSelect + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Aggregate-WindowSelect (stores: []) + --> Aggregate-Project + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- Aggregate-Aggregate-WindowSelect + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000013 (topic: S2) + <-- KTABLE-TOSTREAM-0000000012 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_non-KAFKA_key_format/6.2.0_1608159570437/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_non-KAFKA_key_format/6.2.0_1608159570437/plan.json new file mode 100644 index 000000000000..4fb64ab00fd5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_non-KAFKA_key_format/6.2.0_1608159570437/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (A DECIMAL(4, 2)) WITH (FORMAT='JSON', KAFKA_TOPIC='INPUT');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`A` DECIMAL(4, 2)", + "topicName" : "INPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.A A,\n COUNT(*) COUNT\nFROM INPUT INPUT\nWINDOW HOPPING ( SIZE 30 SECONDS , ADVANCE BY 15 SECONDS ) \nGROUP BY INPUT.A\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`A` DECIMAL(4, 2) KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "HOPPING", + "size" : 30.000000000 + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamWindowedAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "INPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`A` DECIMAL(4, 2)" + }, + "selectExpressions" : [ "A AS A", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "A" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "A", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ], + "windowExpression" : " HOPPING ( SIZE 30 SECONDS , ADVANCE BY 15 SECONDS ) " + }, + "keyColumnNames" : [ "A" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_non-KAFKA_key_format/6.2.0_1608159570437/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_non-KAFKA_key_format/6.2.0_1608159570437/spec.json new file mode 100644 index 000000000000..b53a3932dbd0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_non-KAFKA_key_format/6.2.0_1608159570437/spec.json @@ -0,0 +1,195 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159570437, + "path" : "query-validation-tests/hopping-windows.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`A` DECIMAL(4, 2) KEY, `A` DECIMAL(4, 2), `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`A` DECIMAL(4, 2)", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`A` DECIMAL(4, 2) KEY, `A` DECIMAL(4, 2), `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`A` DECIMAL(4, 2) KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 30.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "non-KAFKA key format", + "inputs" : [ { + "topic" : "INPUT", + "key" : null, + "value" : { + "A" : 12.30 + }, + "timestamp" : 10 + }, { + "topic" : "INPUT", + "key" : null, + "value" : { + "A" : 12.30 + }, + "timestamp" : 11 + }, { + "topic" : "INPUT", + "key" : null, + "value" : { + "A" : 1.00 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 12.30, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 10, + "window" : { + "start" : 0, + "end" : 30000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : 12.30, + "value" : { + "COUNT" : 2 + }, + "timestamp" : 11, + "window" : { + "start" : 0, + "end" : 30000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : 1.00, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 12, + "window" : { + "start" : 0, + "end" : 30000, + "type" : "TIME" + } + } ], + "topics" : [ { + "name" : "INPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (A DECIMAL(4,2)) WITH (kafka_topic='INPUT', format='JSON');", "CREATE TABLE OUTPUT AS SELECT A, COUNT() AS COUNT FROM INPUT WINDOW HOPPING (SIZE 30 SECONDS, ADVANCE BY 15 SECONDS) group by A;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`A` DECIMAL(4, 2)", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`A` DECIMAL(4, 2) KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "windowType" : "HOPPING", + "windowSize" : 30000 + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "INPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 30.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 30.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_non-KAFKA_key_format/6.2.0_1608159570437/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_non-KAFKA_key_format/6.2.0_1608159570437/topology new file mode 100644 index 000000000000..e8e67ad1d5a5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/hopping-windows_-_non-KAFKA_key_format/6.2.0_1608159570437/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [INPUT]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Aggregate-WindowSelect + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Aggregate-WindowSelect (stores: []) + --> Aggregate-Project + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- Aggregate-Aggregate-WindowSelect + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000013 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000012 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col/6.2.0_1608159593976/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col/6.2.0_1608159593976/plan.json new file mode 100644 index 000000000000..1b9d7f6f2e10 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col/6.2.0_1608159593976/plan.json @@ -0,0 +1,185 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K INTEGER KEY, K2 INTEGER KEY, V INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.K K,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY INPUT.K\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER" + }, + "keyColumnNames" : [ "K", "K2" ], + "selectExpressions" : [ "K AS K", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "K" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "K", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col/6.2.0_1608159593976/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col/6.2.0_1608159593976/spec.json new file mode 100644 index 000000000000..0f9d382318cd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col/6.2.0_1608159593976/spec.json @@ -0,0 +1,139 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159593976, + "path" : "query-validation-tests/multi-col-keys.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`K` INTEGER KEY, `K` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`K` INTEGER KEY, `K` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "group by one col", + "inputs" : [ { + "topic" : "input_topic", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "V" : 0 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "COUNT" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K INT KEY, K2 INT KEY, V INT) WITH (kafka_topic='input_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT K, COUNT(*) as COUNT FROM INPUT GROUP BY K;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col/6.2.0_1608159593976/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col/6.2.0_1608159593976/topology new file mode 100644 index 000000000000..5144c38f80a3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col/6.2.0_1608159593976/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col_table_with_tombstones/6.2.0_1608159594095/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col_table_with_tombstones/6.2.0_1608159594095/plan.json new file mode 100644 index 000000000000..5105bc653a56 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col_table_with_tombstones/6.2.0_1608159594095/plan.json @@ -0,0 +1,186 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (K INTEGER PRIMARY KEY, K2 INTEGER PRIMARY KEY, V INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.K K,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY INPUT.K\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K", "K2" ], + "selectExpressions" : [ "K AS K", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "K" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "K", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col_table_with_tombstones/6.2.0_1608159594095/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col_table_with_tombstones/6.2.0_1608159594095/spec.json new file mode 100644 index 000000000000..4826794e1432 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col_table_with_tombstones/6.2.0_1608159594095/spec.json @@ -0,0 +1,160 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159594095, + "path" : "query-validation-tests/multi-col-keys.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`K` INTEGER KEY, `K` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`K` INTEGER KEY, `K` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "group by one col table with tombstones", + "inputs" : [ { + "topic" : "input_topic", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "V" : 0 + } + }, { + "topic" : "input_topic", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "COUNT" : 0 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (K INT PRIMARY KEY, K2 INT PRIMARY KEY, V INT) WITH (kafka_topic='input_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT K, COUNT(*) as COUNT FROM INPUT GROUP BY K;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col_table_with_tombstones/6.2.0_1608159594095/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col_table_with_tombstones/6.2.0_1608159594095/topology new file mode 100644 index 000000000000..6e6a85aa7919 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_one_col_table_with_tombstones/6.2.0_1608159594095/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [input_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols/6.2.0_1608159594037/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols/6.2.0_1608159594037/plan.json new file mode 100644 index 000000000000..b2555c1d920e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols/6.2.0_1608159594037/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K INTEGER KEY, K2 INTEGER KEY, V INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.K K,\n INPUT.K2 K2,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY INPUT.K, INPUT.K2\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER" + }, + "keyColumnNames" : [ "K", "K2" ], + "selectExpressions" : [ "K AS K", "K2 AS K2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "K", "K2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "K", "K2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "K", "K2" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols/6.2.0_1608159594037/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols/6.2.0_1608159594037/spec.json new file mode 100644 index 000000000000..8273cb147c0e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols/6.2.0_1608159594037/spec.json @@ -0,0 +1,136 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159594037, + "path" : "query-validation-tests/multi-col-keys.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `K` INTEGER, `K2` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `K` INTEGER, `K2` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "group by two cols", + "inputs" : [ { + "topic" : "input_topic", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "V" : 0 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "COUNT" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K INT KEY, K2 INT KEY, V INT) WITH (kafka_topic='input_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT K, K2, COUNT(*) as COUNT FROM INPUT GROUP BY K, K2;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols/6.2.0_1608159594037/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols/6.2.0_1608159594037/topology new file mode 100644 index 000000000000..5144c38f80a3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols/6.2.0_1608159594037/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_table_with_tombstones/6.2.0_1608159594188/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_table_with_tombstones/6.2.0_1608159594188/plan.json new file mode 100644 index 000000000000..001109159f28 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_table_with_tombstones/6.2.0_1608159594188/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INPUT (K INTEGER PRIMARY KEY, K2 INTEGER PRIMARY KEY, V INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INPUT", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.K K,\n INPUT.K2 K2,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY INPUT.K, INPUT.K2\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "K", "K2" ], + "selectExpressions" : [ "K AS K", "K2 AS K2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "K", "K2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "K", "K2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "K", "K2" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_table_with_tombstones/6.2.0_1608159594188/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_table_with_tombstones/6.2.0_1608159594188/spec.json new file mode 100644 index 000000000000..0d3d29f1a8e3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_table_with_tombstones/6.2.0_1608159594188/spec.json @@ -0,0 +1,160 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159594188, + "path" : "query-validation-tests/multi-col-keys.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `K` INTEGER, `K2` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `K` INTEGER, `K2` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "group by two cols table with tombstones", + "inputs" : [ { + "topic" : "input_topic", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "V" : 0 + } + }, { + "topic" : "input_topic", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "COUNT" : 0 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE INPUT (K INT PRIMARY KEY, K2 INT PRIMARY KEY, V INT) WITH (kafka_topic='input_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT K, K2, COUNT(*) as COUNT FROM INPUT GROUP BY K, K2;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_table_with_tombstones/6.2.0_1608159594188/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_table_with_tombstones/6.2.0_1608159594188/topology new file mode 100644 index 000000000000..6e6a85aa7919 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_table_with_tombstones/6.2.0_1608159594188/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [input_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_with_AS_VALUE_copies/6.2.0_1608159594280/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_with_AS_VALUE_copies/6.2.0_1608159594280/plan.json new file mode 100644 index 000000000000..5bb01e231585 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_with_AS_VALUE_copies/6.2.0_1608159594280/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K INTEGER KEY, K2 INTEGER KEY, V INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.K K,\n INPUT.K2 K2,\n AS_VALUE(INPUT.K) KV,\n AS_VALUE(INPUT.K2) KV2,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY INPUT.K, INPUT.K2\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `KV` INTEGER, `KV2` INTEGER, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER" + }, + "keyColumnNames" : [ "K", "K2" ], + "selectExpressions" : [ "K AS K", "K2 AS K2", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "K", "K2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "K", "K2", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "K", "K2" ], + "selectExpressions" : [ "AS_VALUE(K) AS KV", "AS_VALUE(K2) AS KV2", "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_with_AS_VALUE_copies/6.2.0_1608159594280/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_with_AS_VALUE_copies/6.2.0_1608159594280/spec.json new file mode 100644 index 000000000000..eb2bf5186147 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_with_AS_VALUE_copies/6.2.0_1608159594280/spec.json @@ -0,0 +1,138 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159594280, + "path" : "query-validation-tests/multi-col-keys.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `K` INTEGER, `K2` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `K` INTEGER, `K2` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `KV` INTEGER, `KV2` INTEGER, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "group by two cols with AS_VALUE copies", + "inputs" : [ { + "topic" : "input_topic", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "V" : 0 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "KV" : 1, + "KV2" : 2, + "COUNT" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K INT KEY, K2 INT KEY, V INT) WITH (kafka_topic='input_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT K, K2, AS_VALUE(K) as kv, AS_VALUE(K2) as kv2, COUNT(*) as COUNT FROM INPUT GROUP BY K, K2;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `KV` INTEGER, `KV2` INTEGER, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_with_AS_VALUE_copies/6.2.0_1608159594280/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_with_AS_VALUE_copies/6.2.0_1608159594280/topology new file mode 100644 index 000000000000..5144c38f80a3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_group_by_two_cols_with_AS_VALUE_copies/6.2.0_1608159594280/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_windowed_group_by_one_col/6.2.0_1608159594338/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_windowed_group_by_one_col/6.2.0_1608159594338/plan.json new file mode 100644 index 000000000000..d81c240d6b35 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_windowed_group_by_one_col/6.2.0_1608159594338/plan.json @@ -0,0 +1,190 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K INTEGER KEY, K2 INTEGER KEY, V INTEGER) WITH (FORMAT='JSON', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.K K,\n COUNT(*) COUNT\nFROM INPUT INPUT\nWINDOW TUMBLING ( SIZE 1 SECONDS ) \nGROUP BY INPUT.K\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamWindowedAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER" + }, + "keyColumnNames" : [ "K", "K2" ], + "selectExpressions" : [ "K AS K", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "K" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "K", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ], + "windowExpression" : " TUMBLING ( SIZE 1 SECONDS ) " + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_windowed_group_by_one_col/6.2.0_1608159594338/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_windowed_group_by_one_col/6.2.0_1608159594338/spec.json new file mode 100644 index 000000000000..c655258f2677 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_windowed_group_by_one_col/6.2.0_1608159594338/spec.json @@ -0,0 +1,201 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159594338, + "path" : "query-validation-tests/multi-col-keys.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`K` INTEGER KEY, `K` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`K` INTEGER KEY, `K` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "windowed group by one col", + "inputs" : [ { + "topic" : "input_topic", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "V" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "input_topic", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "V" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "input_topic", + "key" : { + "K" : 1, + "K2" : 2 + }, + "value" : { + "V" : 0 + }, + "timestamp" : 1001 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "COUNT" : 1 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "COUNT" : 2 + }, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "COUNT" : 1 + }, + "window" : { + "start" : 1000, + "end" : 2000, + "type" : "TIME" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K INT KEY, K2 INT KEY, V INT) WITH (kafka_topic='input_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT K, COUNT(*) as COUNT FROM INPUT WINDOW TUMBLING (SIZE 1 SECOND) GROUP BY K;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` INTEGER KEY, `K2` INTEGER KEY, `V` INTEGER", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`K` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "windowType" : "TUMBLING", + "windowSize" : 1000 + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "TUMBLING", + "size" : 1.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_windowed_group_by_one_col/6.2.0_1608159594338/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_windowed_group_by_one_col/6.2.0_1608159594338/topology new file mode 100644 index 000000000000..64996350c574 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-col-keys_-_windowed_group_by_one_col/6.2.0_1608159594338/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Aggregate-WindowSelect + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Aggregate-WindowSelect (stores: []) + --> Aggregate-Project + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- Aggregate-Aggregate-WindowSelect + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000013 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000012 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/none_-_inherited_in_CTAS_-_GROUP_BY/6.2.0_1608159599843/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/none_-_inherited_in_CTAS_-_GROUP_BY/6.2.0_1608159599843/plan.json new file mode 100644 index 000000000000..e1a931a88b48 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/none_-_inherited_in_CTAS_-_GROUP_BY/6.2.0_1608159599843/plan.json @@ -0,0 +1,184 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (FOO INTEGER) WITH (KAFKA_TOPIC='input_topic', KEY_FORMAT='NONE', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "NONE" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.FOO FOO,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY INPUT.FOO\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`FOO` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "NONE" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`FOO` INTEGER" + }, + "selectExpressions" : [ "FOO AS FOO", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "FOO" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "FOO", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "FOO" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/none_-_inherited_in_CTAS_-_GROUP_BY/6.2.0_1608159599843/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/none_-_inherited_in_CTAS_-_GROUP_BY/6.2.0_1608159599843/spec.json new file mode 100644 index 000000000000..91e591233cd8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/none_-_inherited_in_CTAS_-_GROUP_BY/6.2.0_1608159599843/spec.json @@ -0,0 +1,143 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159599843, + "path" : "query-validation-tests/none.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`FOO` INTEGER KEY, `FOO` INTEGER, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`FOO` INTEGER", + "keyFormat" : { + "format" : "NONE" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`FOO` INTEGER KEY, `FOO` INTEGER, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`FOO` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "inherited in CTAS - GROUP BY", + "inputs" : [ { + "topic" : "input_topic", + "key" : null, + "value" : { + "foo" : 22 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 22, + "value" : { + "COUNT" : 1 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', key_format='NONE', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT FOO, COUNT() AS COUNT FROM INPUT GROUP BY FOO;" ], + "properties" : { + "ksql.persistence.default.format.key" : "JSON" + }, + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`FOO` INTEGER", + "keyFormat" : { + "format" : "NONE" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`FOO` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "NONE" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/none_-_inherited_in_CTAS_-_GROUP_BY/6.2.0_1608159599843/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/none_-_inherited_in_CTAS_-_GROUP_BY/6.2.0_1608159599843/topology new file mode 100644 index 000000000000..5144c38f80a3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/none_-_inherited_in_CTAS_-_GROUP_BY/6.2.0_1608159599843/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_IS_NULL_with_null_value/6.2.0_1608159600023/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_IS_NULL_with_null_value/6.2.0_1608159600023/plan.json new file mode 100644 index 000000000000..f761856d9d1d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_IS_NULL_with_null_value/6.2.0_1608159600023/plan.json @@ -0,0 +1,184 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (NAME STRING) WITH (FORMAT='JSON', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`NAME` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n (INPUT.NAME IS NULL) KSQL_COL_0,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY (INPUT.NAME IS NULL)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`KSQL_COL_0` BOOLEAN KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`NAME` STRING" + }, + "selectExpressions" : [ "NAME AS NAME", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "(NAME IS NULL)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "NAME", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "KSQL_COL_0" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_IS_NULL_with_null_value/6.2.0_1608159600023/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_IS_NULL_with_null_value/6.2.0_1608159600023/spec.json new file mode 100644 index 000000000000..0193115fe412 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_IS_NULL_with_null_value/6.2.0_1608159600023/spec.json @@ -0,0 +1,157 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159600023, + "path" : "query-validation-tests/null.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` BOOLEAN KEY, `NAME` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`NAME` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` BOOLEAN KEY, `NAME` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`KSQL_COL_0` BOOLEAN KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "stream GROUP BY IS NULL with null value", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : null, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "NAME" : null + }, + "timestamp" : 2 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "NAME" : "n" + }, + "timestamp" : 3 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : false, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 3 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (NAME STRING) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT NAME IS NULL, COUNT() AS COUNT FROM INPUT GROUP BY (NAME IS NULL);" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`NAME` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`KSQL_COL_0` BOOLEAN KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_IS_NULL_with_null_value/6.2.0_1608159600023/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_IS_NULL_with_null_value/6.2.0_1608159600023/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_IS_NULL_with_null_value/6.2.0_1608159600023/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_with_null_value/6.2.0_1608159599974/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_with_null_value/6.2.0_1608159599974/plan.json new file mode 100644 index 000000000000..dd6e734aef04 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_with_null_value/6.2.0_1608159599974/plan.json @@ -0,0 +1,180 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (NAME STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`NAME` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.NAME NAME,\n COUNT(*) COUNT\nFROM INPUT INPUT\nGROUP BY INPUT.NAME\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`NAME` STRING KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`NAME` STRING" + }, + "selectExpressions" : [ "NAME AS NAME", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "NAME", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "NAME" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_with_null_value/6.2.0_1608159599974/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_with_null_value/6.2.0_1608159599974/spec.json new file mode 100644 index 000000000000..ac2e81490fb6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_with_null_value/6.2.0_1608159599974/spec.json @@ -0,0 +1,144 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159599974, + "path" : "query-validation-tests/null.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`NAME` STRING KEY, `NAME` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`NAME` STRING KEY, `NAME` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`NAME` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "stream GROUP BY with null value", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : null, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "NAME" : null + }, + "timestamp" : 2 + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "NAME" : "n" + }, + "timestamp" : 3 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "n", + "value" : { + "COUNT" : 1 + }, + "timestamp" : 3 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (NAME STRING) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT NAME, COUNT() AS COUNT FROM INPUT GROUP BY NAME;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`NAME` STRING KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_with_null_value/6.2.0_1608159599974/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_with_null_value/6.2.0_1608159599974/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/null_-_stream_GROUP_BY_with_null_value/6.2.0_1608159599974/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_non-KAFKA_key_format/6.2.0_1608159604628/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_non-KAFKA_key_format/6.2.0_1608159604628/plan.json new file mode 100644 index 000000000000..0fabb42ff495 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_non-KAFKA_key_format/6.2.0_1608159604628/plan.json @@ -0,0 +1,188 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (A DECIMAL(4, 2)) WITH (FORMAT='JSON', KAFKA_TOPIC='INPUT');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`A` DECIMAL(4, 2)", + "topicName" : "INPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.A A,\n COUNT(*) COUNT\nFROM INPUT INPUT\nWINDOW SESSION ( 30 SECONDS ) \nGROUP BY INPUT.A\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`A` DECIMAL(4, 2) KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "SESSION" + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamWindowedAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "INPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`A` DECIMAL(4, 2)" + }, + "selectExpressions" : [ "A AS A", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "A" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "A", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ], + "windowExpression" : " SESSION ( 30 SECONDS ) " + }, + "keyColumnNames" : [ "A" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_non-KAFKA_key_format/6.2.0_1608159604628/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_non-KAFKA_key_format/6.2.0_1608159604628/spec.json new file mode 100644 index 000000000000..d6d2d442d674 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_non-KAFKA_key_format/6.2.0_1608159604628/spec.json @@ -0,0 +1,201 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159604628, + "path" : "query-validation-tests/session-windows.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`A` DECIMAL(4, 2) KEY, `A` DECIMAL(4, 2), `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`A` DECIMAL(4, 2)", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`A` DECIMAL(4, 2) KEY, `A` DECIMAL(4, 2), `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`A` DECIMAL(4, 2) KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "non-KAFKA key format", + "inputs" : [ { + "topic" : "INPUT", + "key" : null, + "value" : { + "A" : 12.30 + }, + "timestamp" : 10 + }, { + "topic" : "INPUT", + "key" : null, + "value" : { + "A" : 12.30 + }, + "timestamp" : 11 + }, { + "topic" : "INPUT", + "key" : null, + "value" : { + "A" : 1.00 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 12.30, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 10, + "window" : { + "start" : 10, + "end" : 10, + "type" : "SESSION" + } + }, { + "topic" : "OUTPUT", + "key" : 12.30, + "value" : null, + "timestamp" : 10, + "window" : { + "start" : 10, + "end" : 10, + "type" : "SESSION" + } + }, { + "topic" : "OUTPUT", + "key" : 12.30, + "value" : { + "COUNT" : 2 + }, + "timestamp" : 11, + "window" : { + "start" : 10, + "end" : 11, + "type" : "SESSION" + } + }, { + "topic" : "OUTPUT", + "key" : 1.00, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 12, + "window" : { + "start" : 12, + "end" : 12, + "type" : "SESSION" + } + } ], + "topics" : [ { + "name" : "INPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (A DECIMAL(4,2)) WITH (kafka_topic='INPUT', format='JSON');", "CREATE TABLE OUTPUT AS SELECT A, COUNT() AS COUNT FROM INPUT WINDOW SESSION (30 SECONDS) group by A;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`A` DECIMAL(4, 2)", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`A` DECIMAL(4, 2) KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "windowType" : "SESSION" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "INPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_non-KAFKA_key_format/6.2.0_1608159604628/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_non-KAFKA_key_format/6.2.0_1608159604628/topology new file mode 100644 index 000000000000..e8e67ad1d5a5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_non-KAFKA_key_format/6.2.0_1608159604628/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [INPUT]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Aggregate-WindowSelect + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Aggregate-WindowSelect (stores: []) + --> Aggregate-Project + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- Aggregate-Aggregate-WindowSelect + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000013 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000012 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_session_start_stream/6.2.0_1608159604562/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_session_start_stream/6.2.0_1608159604562/plan.json new file mode 100644 index 000000000000..da501bb61143 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_session_start_stream/6.2.0_1608159604562/plan.json @@ -0,0 +1,280 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM DATA (USER_ID INTEGER) WITH (KAFKA_TOPIC='data', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "DATA", + "schema" : "`USER_ID` INTEGER", + "topicName" : "data", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE SESSIONS AS SELECT\n DATA.USER_ID USER_ID,\n COUNT(DATA.USER_ID) COUNT\nFROM DATA DATA\nWINDOW SESSION ( 5 SECONDS ) \nGROUP BY DATA.USER_ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "SESSIONS", + "schema" : "`USER_ID` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "SESSIONS", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "windowInfo" : { + "type" : "SESSION" + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "DATA" ], + "sink" : "SESSIONS", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "SESSIONS" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamWindowedAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "data", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`USER_ID` INTEGER" + }, + "selectExpressions" : [ "USER_ID AS USER_ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "USER_ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "USER_ID" ], + "aggregationFunctions" : [ "COUNT(USER_ID)" ], + "windowExpression" : " SESSION ( 5 SECONDS ) " + }, + "keyColumnNames" : [ "USER_ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "SESSIONS" + }, + "queryId" : "CTAS_SESSIONS_0" + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM SESSION_STREAM (USER_ID INTEGER KEY, COUNT BIGINT) WITH (KAFKA_TOPIC='SESSIONS', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON', WINDOW_TYPE='Session');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "SESSION_STREAM", + "schema" : "`USER_ID` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "SESSIONS", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "windowInfo" : { + "type" : "SESSION" + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM SESSION_STARTS AS SELECT *\nFROM SESSION_STREAM SESSION_STREAM\nWHERE (SESSION_STREAM.WINDOWSTART = SESSION_STREAM.WINDOWEND)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "SESSION_STARTS", + "schema" : "`USER_ID` INTEGER KEY, `COUNT` BIGINT", + "topicName" : "SESSION_STARTS", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "windowInfo" : { + "type" : "SESSION" + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "SESSION_STREAM" ], + "sink" : "SESSION_STARTS", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "SESSION_STARTS" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamFilterV1", + "properties" : { + "queryContext" : "WhereFilter" + }, + "source" : { + "@type" : "windowedStreamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "SESSIONS", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "windowInfo" : { + "type" : "SESSION" + }, + "sourceSchema" : "`USER_ID` INTEGER KEY, `COUNT` BIGINT" + }, + "filterExpression" : "(WINDOWSTART = WINDOWEND)" + }, + "keyColumnNames" : [ "USER_ID" ], + "selectExpressions" : [ "COUNT AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "SESSION_STARTS" + }, + "queryId" : "CSAS_SESSION_STARTS_1" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_session_start_stream/6.2.0_1608159604562/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_session_start_stream/6.2.0_1608159604562/spec.json new file mode 100644 index 000000000000..5ef5dd297f75 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_session_start_stream/6.2.0_1608159604562/spec.json @@ -0,0 +1,271 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159604562, + "path" : "query-validation-tests/session-windows.json", + "schemas" : { + "CSAS_SESSION_STARTS_1.KsqlTopic.Source" : { + "schema" : "`USER_ID` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA", + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_SESSION_STARTS_1.SESSION_STARTS" : { + "schema" : "`USER_ID` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA", + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "session start stream", + "inputs" : [ { + "topic" : "data", + "key" : null, + "value" : { + "user_id" : 5 + }, + "timestamp" : 0 + }, { + "topic" : "data", + "key" : null, + "value" : { + "user_id" : 5 + }, + "timestamp" : 1000 + }, { + "topic" : "data", + "key" : null, + "value" : { + "user_id" : 5 + }, + "timestamp" : 2000 + }, { + "topic" : "data", + "key" : null, + "value" : { + "user_id" : 5 + }, + "timestamp" : 70000 + } ], + "outputs" : [ { + "topic" : "SESSIONS", + "key" : 5, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 0, + "window" : { + "start" : 0, + "end" : 0, + "type" : "SESSION" + } + }, { + "topic" : "SESSIONS", + "key" : 5, + "value" : null, + "timestamp" : 0, + "window" : { + "start" : 0, + "end" : 0, + "type" : "SESSION" + } + }, { + "topic" : "SESSIONS", + "key" : 5, + "value" : { + "COUNT" : 2 + }, + "timestamp" : 1000, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "SESSION" + } + }, { + "topic" : "SESSIONS", + "key" : 5, + "value" : null, + "timestamp" : 1000, + "window" : { + "start" : 0, + "end" : 1000, + "type" : "SESSION" + } + }, { + "topic" : "SESSIONS", + "key" : 5, + "value" : { + "COUNT" : 3 + }, + "timestamp" : 2000, + "window" : { + "start" : 0, + "end" : 2000, + "type" : "SESSION" + } + }, { + "topic" : "SESSIONS", + "key" : 5, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 70000, + "window" : { + "start" : 70000, + "end" : 70000, + "type" : "SESSION" + } + }, { + "topic" : "SESSION_STARTS", + "key" : 5, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 0, + "window" : { + "start" : 0, + "end" : 0, + "type" : "SESSION" + } + }, { + "topic" : "SESSION_STARTS", + "key" : 5, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 70000, + "window" : { + "start" : 70000, + "end" : 70000, + "type" : "SESSION" + } + } ], + "topics" : [ { + "name" : "data", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "SESSIONS", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "SESSION_STARTS", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM DATA (USER_ID INT) WITH (kafka_topic='data', value_format='JSON');", "CREATE TABLE SESSIONS as SELECT USER_ID, COUNT(USER_ID) AS COUNT FROM DATA WINDOW SESSION (5 SECONDS) group by USER_ID;", "CREATE STREAM SESSION_STREAM (USER_ID INT KEY, COUNT BIGINT) WITH (kafka_topic='SESSIONS', value_format='JSON', window_type='Session');", "CREATE STREAM SESSION_STARTS AS SELECT * FROM SESSION_STREAM WHERE WINDOWSTART = WINDOWEND;" ], + "post" : { + "sources" : [ { + "name" : "DATA", + "type" : "STREAM", + "schema" : "`USER_ID` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "SESSIONS", + "type" : "TABLE", + "schema" : "`USER_ID` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA", + "windowType" : "SESSION" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "SESSION_STARTS", + "type" : "STREAM", + "schema" : "`USER_ID` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA", + "windowType" : "SESSION" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "SESSION_STREAM", + "type" : "STREAM", + "schema" : "`USER_ID` INTEGER KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "KAFKA", + "windowType" : "SESSION" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "SESSION_STARTS", + "keyFormat" : { + "format" : "KAFKA", + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_SESSIONS_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA", + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "SESSIONS", + "keyFormat" : { + "format" : "KAFKA", + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_SESSIONS_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "data", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_session_start_stream/6.2.0_1608159604562/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_session_start_stream/6.2.0_1608159604562/topology new file mode 100644 index 000000000000..bdd496dcbf3f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/session-windows_-_session_start_stream/6.2.0_1608159604562/topology @@ -0,0 +1,16 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [SESSIONS]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> WhereFilter + <-- KSTREAM-SOURCE-0000000000 + Processor: WhereFilter (stores: []) + --> Project + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000004 + <-- WhereFilter + Sink: KSTREAM-SINK-0000000004 (topic: SESSION_STARTS) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/substring_-_in_group_by/6.2.0_1608159606552/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/substring_-_in_group_by/6.2.0_1608159606552/plan.json new file mode 100644 index 000000000000..dcad57731023 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/substring_-_in_group_by/6.2.0_1608159606552/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, SOURCE STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n SUBSTRING(TEST.SOURCE, 0, 2) THING,\n COUNT(*) SUBSTRING\nFROM TEST TEST\nGROUP BY SUBSTRING(TEST.SOURCE, 0, 2)\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`THING` STRING KEY, `SUBSTRING` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`K` STRING KEY, `SOURCE` STRING" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "SOURCE AS SOURCE", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "SUBSTRING(SOURCE, 0, 2)" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "SOURCE", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "THING" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS SUBSTRING" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/substring_-_in_group_by/6.2.0_1608159606552/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/substring_-_in_group_by/6.2.0_1608159606552/spec.json new file mode 100644 index 000000000000..d5a3fb18287e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/substring_-_in_group_by/6.2.0_1608159606552/spec.json @@ -0,0 +1,166 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159606552, + "path" : "query-validation-tests/substring.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`KSQL_COL_0` STRING KEY, `SOURCE` STRING, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`KSQL_COL_0` STRING KEY, `SOURCE` STRING, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`THING` STRING KEY, `SUBSTRING` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "in group by", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : "some string" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "another string" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "some string again" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "another string again" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "some other string" + }, { + "topic" : "test_topic", + "key" : null, + "value" : "the final string" + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "so", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "an", + "value" : "1" + }, { + "topic" : "OUTPUT", + "key" : "so", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "an", + "value" : "2" + }, { + "topic" : "OUTPUT", + "key" : "so", + "value" : "3" + }, { + "topic" : "OUTPUT", + "key" : "th", + "value" : "1" + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT SUBSTRING(source, 0, 2) AS Thing, COUNT(*) AS SUBSTRING FROM TEST GROUP BY SUBSTRING(source, 0, 2);" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`THING` STRING KEY, `SUBSTRING` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/substring_-_in_group_by/6.2.0_1608159606552/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/substring_-_in_group_by/6.2.0_1608159606552/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/substring_-_in_group_by/6.2.0_1608159606552/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.2.0_1608159606740/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.2.0_1608159606740/plan.json new file mode 100644 index 000000000000..03ba2d562349 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.2.0_1608159606740/plan.json @@ -0,0 +1,241 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, TOTAL INTEGER) WITH (KAFKA_TOPIC='T1', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, TOTAL INTEGER) WITH (KAFKA_TOPIC='T2', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T1.ID T1_ID,\n SUM(T2.TOTAL) SUM\nFROM T1 T1\nLEFT OUTER JOIN T2 T2 ON ((T1.ID = T2.ID))\nGROUP BY T1.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T1_ID` BIGINT KEY, `SUM` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "TOTAL AS T1_TOTAL", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "T2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "forceChangelog" : true + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "TOTAL AS T2_TOTAL", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T1_ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "T1_ID AS T1_ID", "T2_TOTAL AS T2_TOTAL" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "groupByExpressions" : [ "T1_ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "nonAggregateColumns" : [ "T1_ID", "T2_TOTAL" ], + "aggregationFunctions" : [ "SUM(T2_TOTAL)" ] + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS SUM" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.2.0_1608159606740/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.2.0_1608159606740/spec.json new file mode 100644 index 000000000000..e3749752067b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.2.0_1608159606740/spec.json @@ -0,0 +1,368 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159606740, + "path" : "query-validation-tests/sum.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`T1_ID` BIGINT KEY, `T1_ID` BIGINT, `T2_TOTAL` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`T1_ID` BIGINT KEY, `T1_ID` BIGINT, `T2_TOTAL` INTEGER, `KSQL_AGG_VARIABLE_0` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T1_ID` BIGINT KEY, `SUM` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "sum int left join of table", + "inputs" : [ { + "topic" : "T1", + "key" : 0, + "value" : { + "total" : 100 + } + }, { + "topic" : "T1", + "key" : 1, + "value" : { + "total" : 101 + } + }, { + "topic" : "T2", + "key" : 0, + "value" : { + "total" : 5 + } + }, { + "topic" : "T2", + "key" : 1, + "value" : { + "total" : 10 + } + }, { + "topic" : "T2", + "key" : 0, + "value" : { + "total" : 20 + } + }, { + "topic" : "T2", + "key" : 0, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "SUM" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 5 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "SUM" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "SUM" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 20 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 0 + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "SUM" : 0 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T1", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "T2", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE t1 (ID BIGINT PRIMARY KEY, TOTAL integer) WITH (kafka_topic='T1', value_format='AVRO');", "CREATE TABLE t2 (ID BIGINT PRIMARY KEY, TOTAL integer) WITH (kafka_topic='T2', value_format='AVRO');", "CREATE TABLE OUTPUT AS SELECT T1.ID, SUM(t2.total) as SUM FROM T1 LEFT JOIN T2 ON (t1.id = t2.id) GROUP BY t1.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`T1_ID` BIGINT KEY, `SUM` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "T1", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "T2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `TOTAL` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "T1", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "T1_ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "T2_TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "T1_ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "T2_TOTAL", + "type" : [ "null", "int" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", "int" ], + "default" : null + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "SUM", + "type" : [ "null", "int" ], + "default" : null + } ] + } + }, { + "name" : "T2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "TOTAL", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.2.0_1608159606740/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.2.0_1608159606740/topology new file mode 100644 index 000000000000..12834a39f65d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/sum_-_sum_int_left_join_of_table/6.2.0_1608159606740/topology @@ -0,0 +1,69 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [T1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [T2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-FILTER-0000000016 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000018 + <-- KTABLE-FILTER-0000000016 + Sink: KSTREAM-SINK-0000000018 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000019 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000020 + Processor: KTABLE-AGGREGATE-0000000020 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000019 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000020 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000023 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000023 (stores: []) + --> KSTREAM-SINK-0000000024 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000024 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000023 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.2.0_1608159610396/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.2.0_1608159610396/plan.json new file mode 100644 index 000000000000..ca219d9cf005 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.2.0_1608159610396/plan.json @@ -0,0 +1,182 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `REGION` STRING", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE SUM_ID_BY_REGION AS SELECT\n TEST.REGION REGION,\n TEST_UDAF(TEST.ID) KSQL_COL_0\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "SUM_ID_BY_REGION", + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` BIGINT", + "topicName" : "SUM_ID_BY_REGION", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "SUM_ID_BY_REGION", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "SUM_ID_BY_REGION" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `REGION` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "REGION AS REGION", "ID AS ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "nonAggregateColumns" : [ "REGION", "ID" ], + "aggregationFunctions" : [ "TEST_UDAF(ID)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "topicName" : "SUM_ID_BY_REGION" + }, + "queryId" : "CTAS_SUM_ID_BY_REGION_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.2.0_1608159610396/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.2.0_1608159610396/spec.json new file mode 100644 index 000000000000..b52b9955e3f9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.2.0_1608159610396/spec.json @@ -0,0 +1,178 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159610396, + "path" : "query-validation-tests/test-custom-udaf.json", + "schemas" : { + "CTAS_SUM_ID_BY_REGION_0.Aggregate.GroupBy" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_SUM_ID_BY_REGION_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `ID` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_SUM_ID_BY_REGION_0.SUM_ID_BY_REGION" : { + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, + "CTAS_SUM_ID_BY_REGION_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } + }, + "testCase" : { + "name" : "test_udaf on a table", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : "alice,east" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : "bob,east" + }, { + "topic" : "test_topic", + "key" : 2, + "value" : "carol,west" + }, { + "topic" : "test_topic", + "key" : 3, + "value" : "dave,west" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : "bob,west" + }, { + "topic" : "test_topic", + "key" : 1, + "value" : null + } ], + "outputs" : [ { + "topic" : "SUM_ID_BY_REGION", + "key" : "east", + "value" : "0" + }, { + "topic" : "SUM_ID_BY_REGION", + "key" : "east", + "value" : "1" + }, { + "topic" : "SUM_ID_BY_REGION", + "key" : "west", + "value" : "2" + }, { + "topic" : "SUM_ID_BY_REGION", + "key" : "west", + "value" : "5" + }, { + "topic" : "SUM_ID_BY_REGION", + "key" : "east", + "value" : "0" + }, { + "topic" : "SUM_ID_BY_REGION", + "key" : "west", + "value" : "6" + }, { + "topic" : "SUM_ID_BY_REGION", + "key" : "west", + "value" : "5" + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "SUM_ID_BY_REGION", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, REGION string) WITH (kafka_topic='test_topic', value_format='DELIMITED');", "CREATE TABLE SUM_ID_BY_REGION AS SELECT REGION, test_udaf(id) FROM TEST GROUP BY REGION;" ], + "post" : { + "sources" : [ { + "name" : "SUM_ID_BY_REGION", + "type" : "TABLE", + "schema" : "`REGION` STRING KEY, `KSQL_COL_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "SUM_ID_BY_REGION", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_SUM_ID_BY_REGION_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_SUM_ID_BY_REGION_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_SUM_ID_BY_REGION_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.2.0_1608159610396/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.2.0_1608159610396/topology new file mode 100644 index 000000000000..2709559cf198 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_on_a_table/6.2.0_1608159610396/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-MAPVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: SUM_ID_BY_REGION) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_with_struct/6.2.0_1608159610542/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_with_struct/6.2.0_1608159610542/plan.json new file mode 100644 index 000000000000..7c967e25684e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_with_struct/6.2.0_1608159610542/plan.json @@ -0,0 +1,181 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K BIGINT KEY, ID STRING, VAL STRUCT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` BIGINT KEY, `ID` STRING, `VAL` STRUCT<`A` INTEGER, `B` INTEGER>", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RESULT AS SELECT\n TEST.ID ID,\n TEST_UDAF(TEST.VAL) RESULT\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RESULT", + "schema" : "`ID` STRING KEY, `RESULT` STRUCT<`A` INTEGER, `B` INTEGER>", + "topicName" : "RESULT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "RESULT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "RESULT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` BIGINT KEY, `ID` STRING, `VAL` STRUCT<`A` INTEGER, `B` INTEGER>" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "ID AS ID", "VAL AS VAL" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "ID", "VAL" ], + "aggregationFunctions" : [ "TEST_UDAF(VAL)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS RESULT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "RESULT" + }, + "queryId" : "CTAS_RESULT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_with_struct/6.2.0_1608159610542/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_with_struct/6.2.0_1608159610542/spec.json new file mode 100644 index 000000000000..072cf1f5c13d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_with_struct/6.2.0_1608159610542/spec.json @@ -0,0 +1,175 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159610542, + "path" : "query-validation-tests/test-custom-udaf.json", + "schemas" : { + "CTAS_RESULT_0.RESULT" : { + "schema" : "`ID` STRING KEY, `RESULT` STRUCT<`A` INTEGER, `B` INTEGER>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_RESULT_0.Aggregate.GroupBy" : { + "schema" : "`ID` STRING KEY, `ID` STRING, `VAL` STRUCT<`A` INTEGER, `B` INTEGER>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_RESULT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` STRING KEY, `ID` STRING, `VAL` STRUCT<`A` INTEGER, `B` INTEGER>, `KSQL_AGG_VARIABLE_0` STRUCT<`A` INTEGER, `B` INTEGER>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_RESULT_0.KsqlTopic.Source" : { + "schema" : "`K` BIGINT KEY, `ID` STRING, `VAL` STRUCT<`A` INTEGER, `B` INTEGER>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "test_udaf with struct", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "id" : "0", + "val" : { + "A" : 1, + "B" : 2 + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "id" : "0", + "val" : { + "A" : 2, + "B" : 3 + } + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "id" : "1", + "val" : { + "A" : 1, + "B" : 0 + } + } + } ], + "outputs" : [ { + "topic" : "RESULT", + "key" : "0", + "value" : { + "RESULT" : { + "A" : 1, + "B" : 2 + } + } + }, { + "topic" : "RESULT", + "key" : "0", + "value" : { + "RESULT" : { + "A" : 3, + "B" : 5 + } + } + }, { + "topic" : "RESULT", + "key" : "1", + "value" : { + "RESULT" : { + "A" : 1, + "B" : 0 + } + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "RESULT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K BIGINT KEY, id VARCHAR, val STRUCT) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE TABLE RESULT AS SELECT ID, test_udaf(val) as result FROM TEST GROUP BY ID;" ], + "post" : { + "sources" : [ { + "name" : "RESULT", + "type" : "TABLE", + "schema" : "`ID` STRING KEY, `RESULT` STRUCT<`A` INTEGER, `B` INTEGER>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` BIGINT KEY, `ID` STRING, `VAL` STRUCT<`A` INTEGER, `B` INTEGER>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_RESULT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_RESULT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "RESULT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_with_struct/6.2.0_1608159610542/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_with_struct/6.2.0_1608159610542/topology new file mode 100644 index 000000000000..943e16230e4b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/test-custom-udaf_-_test_udaf_with_struct/6.2.0_1608159610542/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: RESULT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/timestamp-to-string_-_timestamp_to_string_in_join/6.2.0_1608159611149/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/timestamp-to-string_-_timestamp_to_string_in_join/6.2.0_1608159611149/plan.json new file mode 100644 index 000000000000..3c46289fca80 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/timestamp-to-string_-_timestamp_to_string_in_join/6.2.0_1608159611149/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K STRING KEY, WLAN_SA ARRAY) WITH (KAFKA_TOPIC='input', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` STRING KEY, `WLAN_SA` ARRAY", + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE DEVICES (ROWKEY STRING PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='devices', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "DEVICES", + "schema" : "`ROWKEY` STRING KEY, `NAME` STRING", + "topicName" : "devices", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n D.NAME DEVICE_NAME,\n TIMESTAMPTOSTRING(MIN(P.ROWTIME), 'yyyy-MM-dd HH:mm:ss zzz', 'Europe/London') KSQL_COL_0\nFROM INPUT P\nINNER JOIN DEVICES D ON ((P.WLAN_SA[1] = D.ROWKEY))\nGROUP BY D.NAME\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`DEVICE_NAME` STRING KEY, `KSQL_COL_0` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "DEVICES", "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "input", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`K` STRING KEY, `WLAN_SA` ARRAY" + }, + "keyExpression" : "WLAN_SA[1]" + }, + "keyColumnNames" : [ "P_KSQL_COL_0" ], + "selectExpressions" : [ "WLAN_SA AS P_WLAN_SA", "ROWTIME AS P_ROWTIME", "K AS P_K", "KSQL_COL_0 AS P_KSQL_COL_0" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "devices", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ROWKEY` STRING KEY, `NAME` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "D_ROWKEY" ], + "selectExpressions" : [ "NAME AS D_NAME", "ROWTIME AS D_ROWTIME", "ROWKEY AS D_ROWKEY" ] + }, + "keyColName" : "D_ROWKEY" + }, + "keyColumnNames" : [ "D_ROWKEY" ], + "selectExpressions" : [ "D_NAME AS D_NAME", "P_ROWTIME AS P_ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "groupByExpressions" : [ "D_NAME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "nonAggregateColumns" : [ "D_NAME", "P_ROWTIME" ], + "aggregationFunctions" : [ "MIN(P_ROWTIME)" ] + }, + "keyColumnNames" : [ "DEVICE_NAME" ], + "selectExpressions" : [ "TIMESTAMPTOSTRING(KSQL_AGG_VARIABLE_0, 'yyyy-MM-dd HH:mm:ss zzz', 'Europe/London') AS KSQL_COL_0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/timestamp-to-string_-_timestamp_to_string_in_join/6.2.0_1608159611149/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/timestamp-to-string_-_timestamp_to_string_in_join/6.2.0_1608159611149/spec.json new file mode 100644 index 000000000000..06a25a3f71d8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/timestamp-to-string_-_timestamp_to_string_in_join/6.2.0_1608159611149/spec.json @@ -0,0 +1,199 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159611149, + "path" : "query-validation-tests/timestamp-to-string.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`D_NAME` STRING KEY, `D_NAME` STRING, `P_ROWTIME` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join.Left" : { + "schema" : "`P_KSQL_COL_0` STRING KEY, `P_WLAN_SA` ARRAY, `P_ROWTIME` BIGINT, `P_K` STRING, `P_KSQL_COL_0` STRING", + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`P_KSQL_COL_0` STRING KEY, `P_WLAN_SA` ARRAY, `P_ROWTIME` BIGINT, `P_K` STRING, `P_KSQL_COL_0` STRING", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`K` STRING KEY, `WLAN_SA` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ROWKEY` STRING KEY, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`D_NAME` STRING KEY, `D_NAME` STRING, `P_ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`DEVICE_NAME` STRING KEY, `KSQL_COL_0` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "timestamp to string in join", + "inputs" : [ { + "topic" : "devices", + "key" : "a", + "value" : { + "name" : "device" + }, + "timestamp" : 1526075912000 + }, { + "topic" : "input", + "key" : "foo", + "value" : { + "WLAN_SA" : [ "a" ] + }, + "timestamp" : 1526075913000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "device", + "value" : { + "KSQL_COL_0" : "2018-05-11 22:58:33 BST" + }, + "timestamp" : 1526075913000 + } ], + "topics" : [ { + "name" : "input", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "devices", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K STRING KEY, WLAN_SA ARRAY) WITH (kafka_topic='input', value_format='JSON');", "CREATE TABLE DEVICES (ROWKEY STRING PRIMARY KEY, NAME VARCHAR) WITH (kafka_topic='devices', value_format='JSON');", "CREATE TABLE OUTPUT AS SELECT D.NAME AS DEVICE_NAME, TIMESTAMPTOSTRING(MIN(P.ROWTIME),'yyyy-MM-dd HH:mm:ss zzz','Europe/London') FROM INPUT P INNER JOIN DEVICES D ON P.WLAN_SA[1] = D.ROWKEY GROUP BY D.NAME emit changes;" ], + "post" : { + "sources" : [ { + "name" : "DEVICES", + "type" : "TABLE", + "schema" : "`ROWKEY` STRING KEY, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `WLAN_SA` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`DEVICE_NAME` STRING KEY, `KSQL_COL_0` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Join-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "input", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "devices", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/timestamp-to-string_-_timestamp_to_string_in_join/6.2.0_1608159611149/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/timestamp-to-string_-_timestamp_to_string_in_join/6.2.0_1608159611149/topology new file mode 100644 index 000000000000..35debdcc8870 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/timestamp-to-string_-_timestamp_to_string_in_join/6.2.0_1608159611149/topology @@ -0,0 +1,72 @@ +Topologies: + Sub-topology: 0 + Source: Join-repartition-source (topics: [Join-repartition]) + --> Join + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Aggregate-Prepare + <-- Join-repartition-source + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000015 + <-- Join + Source: KSTREAM-SOURCE-0000000001 (topics: [devices]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-FILTER-0000000015 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000015 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000003 + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000006 (topics: [input]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000006 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: PrependAliasLeft (stores: []) + --> Join-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-repartition-filter (stores: []) + --> Join-repartition-sink + <-- PrependAliasLeft + Sink: Join-repartition-sink (topic: Join-repartition) + <-- Join-repartition-filter + + Sub-topology: 2 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000017 + Processor: KSTREAM-AGGREGATE-0000000017 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000017 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000023 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000023 (stores: []) + --> KSTREAM-SINK-0000000024 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000024 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000023 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/tumbling-windows_-_non-KAFKA_key_format/6.2.0_1608159613389/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/tumbling-windows_-_non-KAFKA_key_format/6.2.0_1608159613389/plan.json new file mode 100644 index 000000000000..9777180c8c09 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/tumbling-windows_-_non-KAFKA_key_format/6.2.0_1608159613389/plan.json @@ -0,0 +1,189 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (A DECIMAL(4, 2)) WITH (FORMAT='JSON', KAFKA_TOPIC='INPUT');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`A` DECIMAL(4, 2)", + "topicName" : "INPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n INPUT.A A,\n COUNT(*) COUNT\nFROM INPUT INPUT\nWINDOW TUMBLING ( SIZE 30 SECONDS ) \nGROUP BY INPUT.A\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`A` DECIMAL(4, 2) KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "TUMBLING", + "size" : 30.000000000 + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamWindowedAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "INPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`A` DECIMAL(4, 2)" + }, + "selectExpressions" : [ "A AS A", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "A" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "A", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ], + "windowExpression" : " TUMBLING ( SIZE 30 SECONDS ) " + }, + "keyColumnNames" : [ "A" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/tumbling-windows_-_non-KAFKA_key_format/6.2.0_1608159613389/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/tumbling-windows_-_non-KAFKA_key_format/6.2.0_1608159613389/spec.json new file mode 100644 index 000000000000..6157b995e8d9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/tumbling-windows_-_non-KAFKA_key_format/6.2.0_1608159613389/spec.json @@ -0,0 +1,195 @@ +{ + "version" : "6.2.0", + "timestamp" : 1608159613389, + "path" : "query-validation-tests/tumbling-windows.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`A` DECIMAL(4, 2) KEY, `A` DECIMAL(4, 2), `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`A` DECIMAL(4, 2)", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`A` DECIMAL(4, 2) KEY, `A` DECIMAL(4, 2), `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`A` DECIMAL(4, 2) KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "TUMBLING", + "size" : 30.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "non-KAFKA key format", + "inputs" : [ { + "topic" : "INPUT", + "key" : null, + "value" : { + "A" : 12.30 + }, + "timestamp" : 10 + }, { + "topic" : "INPUT", + "key" : null, + "value" : { + "A" : 12.30 + }, + "timestamp" : 11 + }, { + "topic" : "INPUT", + "key" : null, + "value" : { + "A" : 1.00 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 12.30, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 10, + "window" : { + "start" : 0, + "end" : 30000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : 12.30, + "value" : { + "COUNT" : 2 + }, + "timestamp" : 11, + "window" : { + "start" : 0, + "end" : 30000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : 1.00, + "value" : { + "COUNT" : 1 + }, + "timestamp" : 12, + "window" : { + "start" : 0, + "end" : 30000, + "type" : "TIME" + } + } ], + "topics" : [ { + "name" : "INPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (A DECIMAL(4,2)) WITH (kafka_topic='INPUT', format='JSON');", "CREATE TABLE OUTPUT AS SELECT A, COUNT() AS COUNT FROM INPUT WINDOW TUMBLING (SIZE 30 SECONDS) group by A;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`A` DECIMAL(4, 2)", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`A` DECIMAL(4, 2) KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "JSON", + "windowType" : "TUMBLING", + "windowSize" : 30000 + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "INPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "TUMBLING", + "size" : 30.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "TUMBLING", + "size" : 30.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/tumbling-windows_-_non-KAFKA_key_format/6.2.0_1608159613389/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/tumbling-windows_-_non-KAFKA_key_format/6.2.0_1608159613389/topology new file mode 100644 index 000000000000..e8e67ad1d5a5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/tumbling-windows_-_non-KAFKA_key_format/6.2.0_1608159613389/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [INPUT]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Aggregate-WindowSelect + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Aggregate-WindowSelect (stores: []) + --> Aggregate-Project + <-- Aggregate-Aggregate-ToOutputSchema + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- Aggregate-Aggregate-WindowSelect + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000013 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000012 + diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/delimited.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/delimited.json index 8c98601c2428..d32eb858f541 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/delimited.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/delimited.json @@ -109,9 +109,6 @@ }, { "name": "multi-column key", - "properties": { - "ksql.multicol.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K1 STRING KEY, K2 STRING KEY, V INT) WITH (kafka_topic='input_topic', format='DELIMITED');", "CREATE STREAM OUTPUT WITH (key_format='JSON') AS SELECT * FROM INPUT;" @@ -129,9 +126,6 @@ }, { "name": "multi-column key with different kv delimiters", - "properties": { - "ksql.multicol.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K1 STRING KEY, K2 STRING KEY, V INT, V2 INT) WITH (kafka_topic='input_topic', format='DELIMITED', key_delimiter=';', value_delimiter='-');", "CREATE STREAM OUTPUT WITH (key_format='JSON') AS SELECT * FROM INPUT;" diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/group-by-multi-col.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/group-by-multi-col.json deleted file mode 100644 index 5cf3dcf2413f..000000000000 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/group-by-multi-col.json +++ /dev/null @@ -1,3051 +0,0 @@ -{ - "tests": [ - { - "name": "only key column (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (ID INT KEY, IGNORED INT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT ID, COUNT(1) AS COUNT FROM INPUT GROUP BY ID;" - ], - "inputs": [ - {"topic": "test_topic", "timestamp": 12345, "key": 11, "value": {}}, - {"topic": "test_topic", "timestamp": 12365, "key": 10, "value": {}}, - {"topic": "test_topic", "timestamp": 12375, "key": 11, "value": {}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 11, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 10, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 11, "value": {"COUNT": 2}} - ], - "post": { - "topics": { - "blacklist": ".*-repartition" - }, - "sources": [ - { - "name": "OUTPUT", - "type": "table", - "keyFormat": {"format": "KAFKA"}, - "schema": "ID INT KEY, COUNT BIGINT" - } - ] - } - }, - { - "name": "udafs only in having (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (NAME STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT NAME, LEN(NAME) AS LEN FROM INPUT GROUP BY NAME HAVING COUNT(NAME) = 2;" - ], - "inputs": [ - {"topic": "test_topic", "value": {"NAME": "bob"}, "timestamp": 1}, - {"topic": "test_topic", "value": {"NAME": "bob"}, "timestamp": 2}, - {"topic": "test_topic", "value": {"NAME": "bob"}, "timestamp": 3} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "bob", "value": {"LEN": 3}, "timestamp": 2}, - {"topic": "OUTPUT", "key": "bob", "value": null} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "keyFormat": {"format": "KAFKA"}, "schema": "NAME STRING KEY, LEN INT"} - ] - } - }, - { - "name": "all columns - repartition (stream->table)", - "comment": "Currently, at least one value column is required...", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (NAME STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT NAME FROM INPUT GROUP BY NAME HAVING COUNT(NAME) = 1;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The projection contains no value columns." - } - }, - { - "name": "all columns - no repartition (stream->table)", - "comment": "Currently, at least one value column is required...", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (NAME STRING KEY, V0 INT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT NAME FROM INPUT GROUP BY NAME HAVING COUNT(NAME) = 1;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The projection contains no value columns." - } - }, - { - "name": "value column (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) AS COUNT FROM TEST GROUP BY DATA;" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"data": "d1"}}, - {"topic": "test_topic", "key": 1, "value": {"data": "d2"}}, - {"topic": "test_topic", "key": 2, "value": {"data": "d1"}}, - {"topic": "test_topic", "key": 3, "value": {"data": "d2"}}, - {"topic": "test_topic", "key": 4, "value": {"data": "d1"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d1", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "d2", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "d1", "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": "d2", "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": "d1", "value": {"COUNT": 3}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "DATA STRING KEY, COUNT BIGINT"} - ] - } - }, - { - "name": "struct field (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, ADDRESS STRUCT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT ADDRESS->TOWN, COUNT(*) AS COUNT FROM TEST GROUP BY ADDRESS->TOWN;" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"ADDRESS": {"STREET": "1st Steet", "Town": "Oxford"}}}, - {"topic": "test_topic", "key": 1, "value": {"ADDRESS": {"STREET": "1st Steet", "Town": "London"}}}, - {"topic": "test_topic", "key": 2, "value": {"ADDRESS": {"STREET": "1st Steet", "Town": "Oxford"}}}, - {"topic": "test_topic", "key": 3, "value": {"ADDRESS": {"STREET": "1st Steet", "Town": "London"}}}, - {"topic": "test_topic", "key": 4, "value": {"ADDRESS": {"STREET": "1st Steet", "Town": "Oxford"}}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "Oxford", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "London", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "Oxford", "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": "London", "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": "Oxford", "value": {"COUNT": 3}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "TOWN STRING KEY, COUNT BIGINT"} - ] - } - }, - { - "name": "single expression (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, DATA STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT LEN(DATA), COUNT(*) AS COUNT FROM TEST GROUP BY LEN(DATA);" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"data": "22"}}, - {"topic": "test_topic", "key": 1, "value": {"data": "333"}}, - {"topic": "test_topic", "key": 2, "value": {"data": "-2"}}, - {"topic": "test_topic", "key": 3, "value": {"data": "003"}}, - {"topic": "test_topic", "key": 4, "value": {"data": "2-"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 3, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": 3, "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 3}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_0 INT KEY, COUNT BIGINT"} - ] - } - }, - { - "name": "single column with alias (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT DATA AS NEW_KEY, COUNT(*) AS COUNT FROM TEST GROUP BY DATA;" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"data": "d1"}}, - {"topic": "test_topic", "key": 1, "value": {"data": "d2"}}, - {"topic": "test_topic", "key": 2, "value": {"data": "d1"}}, - {"topic": "test_topic", "key": 3, "value": {"data": "d2"}}, - {"topic": "test_topic", "key": 4, "value": {"data": "d1"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d1", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "d2", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "d1", "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": "d2", "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": "d1", "value": {"COUNT": 3}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "NEW_KEY STRING KEY, COUNT BIGINT"} - ] - } - }, - { - "name": "single column with alias (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT DATA AS NEW_KEY, COUNT(*) AS COUNT FROM TEST GROUP BY DATA;" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"data": "d1"}}, - {"topic": "test_topic", "key": 1, "value": {"data": "d2"}}, - {"topic": "test_topic", "key": 2, "value": {"data": "d1"}}, - {"topic": "test_topic", "key": 3, "value": {"data": "d2"}}, - {"topic": "test_topic", "key": 4, "value": {"data": "d1"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d1", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "d2", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "d1", "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": "d2", "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": "d1", "value": {"COUNT": 3}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "NEW_KEY STRING KEY, COUNT BIGINT"} - ] - } - }, - { - "name": "single expression with alias (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT LEN(DATA) AS NEW_KEY, COUNT(*) AS COUNT FROM TEST GROUP BY LEN(DATA);" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"data": "22"}}, - {"topic": "test_topic", "key": 1, "value": {"data": "333"}}, - {"topic": "test_topic", "key": 2, "value": {"data": "-2"}}, - {"topic": "test_topic", "key": 3, "value": {"data": "003"}}, - {"topic": "test_topic", "key": 4, "value": {"data": "2-"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 3, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": 3, "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 3}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "NEW_KEY INT KEY, COUNT BIGINT"} - ] - } - }, - { - "name": "single expression with alias (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT LEN(DATA) AS NEW_KEY, COUNT(*) AS COUNT FROM TEST GROUP BY LEN(DATA);" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"data": "22"}}, - {"topic": "test_topic", "key": 1, "value": {"data": "333"}}, - {"topic": "test_topic", "key": 2, "value": {"data": "-2"}}, - {"topic": "test_topic", "key": 3, "value": {"data": "003"}}, - {"topic": "test_topic", "key": 4, "value": {"data": "2-"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 3, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": 3, "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 3}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "NEW_KEY INT KEY, COUNT BIGINT"} - ] - } - }, - { - "name": "steam with no key", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (data INT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) AS COUNT FROM TEST GROUP BY DATA;" - ], - "inputs": [ - {"topic": "test_topic", "value": {"data": 22}}, - {"topic": "test_topic", "value": {"data": 333}}, - {"topic": "test_topic", "value": {"data": 22}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 22, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 333, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 22, "value": {"COUNT": 2}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "DATA INT KEY, COUNT BIGINT"} - ] - } - }, - { - "name": "subscript in group-by and select", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 MAP) WITH (kafka_topic='test_topic', value_format='json');", - "CREATE TABLE OUTPUT AS SELECT col1['foo'] AS NEW_KEY, COUNT(*) AS COUNT FROM INPUT GROUP BY col1['foo'];" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"col1": {"foo" : "lala"}}}, - {"topic": "test_topic", "key": 1, "value": {"col1": {"foo" : "kaka"}}}, - {"topic": "test_topic", "key": 2, "value": {"col1": {"alice" : "wonderland"}}}, - {"topic": "test_topic", "key": 3, "value": {"col1": {"mary" : "lamb"}}}, - {"topic": "test_topic", "key": 4, "value": {"col1": null}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "lala", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "kaka", "value": {"COUNT": 1}} - ] - }, - { - "name": "subscript in group-by and having", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 MAP) WITH (kafka_topic='test_topic', value_format='json');", - "CREATE TABLE OUTPUT AS SELECT col1['foo'] AS NEW_KEY, COUNT(*) AS COUNT FROM INPUT GROUP BY col1['foo'] HAVING col1['foo']='lala';" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"col1": {"foo" : "lala"}}}, - {"topic": "test_topic", "key": 1, "value": {"col1": {"foo" : "kaka"}}}, - {"topic": "test_topic", "key": 2, "value": {"col1": {"alice" : "wonderland"}}}, - {"topic": "test_topic", "key": 3, "value": {"col1": {"mary" : "lamb"}}}, - {"topic": "test_topic", "key": 4, "value": {"col1": null}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "lala", "value": {"COUNT": 1}} - ] - }, - { - "name": "subscript in group-by and non aggregate function in select", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 MAP) WITH (kafka_topic='test_topic', value_format='json');", - "CREATE TABLE OUTPUT AS SELECT col1['foo'] AS NEW_KEY, AS_VALUE(col1['foo']) as VV, COUNT(*) AS COUNT FROM INPUT GROUP BY col1['foo'];" - - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"col1": {"foo" : "lala"}}}, - {"topic": "test_topic", "key": 1, "value": {"col1": {"foo" : "kaka"}}}, - {"topic": "test_topic", "key": 2, "value": {"col1": {"alice" : "wonderland"}}}, - {"topic": "test_topic", "key": 3, "value": {"col1": {"mary" : "lamb"}}}, - {"topic": "test_topic", "key": 4, "value": {"col1": null}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "lala", "value": {"VV": "lala","COUNT": 1}}, - {"topic": "OUTPUT", "key": "kaka", "value": {"VV": "kaka","COUNT": 1}} - ] - }, - { - "name": "struct in group-by and non aggregate function in select", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 STRUCT) WITH (kafka_topic='test_topic', value_format='json');", - "CREATE TABLE OUTPUT AS SELECT col1->a AS NEW_KEY, AS_VALUE(col1->a) as VV, COUNT(*) AS COUNT FROM INPUT GROUP BY col1->a;" - - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"col1": {"a" : "lala", "b": 1}}}, - {"topic": "test_topic", "key": 1, "value": {"col1": {"a" : "lala", "b": 2}}}, - {"topic": "test_topic", "key": 2, "value": {"col1": {"a" : "wonderland", "b": 3}}}, - {"topic": "test_topic", "key": 3, "value": {"col1": {"a" : "lamb", "b": 4}}}, - {"topic": "test_topic", "key": 4, "value": {"col1": null}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "lala", "value": {"VV": "lala","COUNT": 1}}, - {"topic": "OUTPUT", "key": "lala", "value": {"VV": "lala","COUNT": 2}}, - {"topic": "OUTPUT", "key": "wonderland", "value": {"VV": "wonderland","COUNT": 1}}, - {"topic": "OUTPUT", "key": "lamb", "value": {"VV": "lamb","COUNT": 1}} - ] - }, - { - "name": "function in group-by and nested function in select", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', format='json');", - "CREATE TABLE OUTPUT AS SELECT INITCAP(COL1) AS G1, COL2 AS G2, TRIM(COL3) AS G3, concat(initcap(col1), col2, trim(col3)) AS foo, COUNT(*) FROM input GROUP BY INITCAP(col1), col2, TRIM(col3);" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"col1": "smells", "col2": "like", "col3": "teen spirit"}}, - {"topic": "test_topic", "key": 1, "value": {"col1": "the", "col2": "man who", "col3": "stole the world"}}, - {"topic": "test_topic", "key": 2, "value": {"col1": "smells", "col2": "like", "col3": "spring"}}, - {"topic": "test_topic", "key": 3, "value": {"col1": "smells", "col2": "like", "col3": " teen spirit "}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "teen spirit"}, "value": {"FOO": "Smellsliketeen spirit", "KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"G1": "The", "G2": "man who", "G3": "stole the world"}, "value": {"FOO": "Theman whostole the world", "KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "spring"}, "value": {"FOO": "Smellslikespring","KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "teen spirit"}, "value": {"FOO": "Smellsliketeen spirit","KSQL_COL_0": 2}} - ] - }, - { - "name": "group by column in nested non-aggregate function in select", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', format='json');", - "CREATE TABLE OUTPUT AS SELECT INITCAP(COL1) AS G1, COL2 AS G2, COL3 AS G3, concat(initcap(col1), col2, trim(col3)) AS foo, COUNT(*) FROM input GROUP BY INITCAP(col1), col2, col3;" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"col1": "smells", "col2": "like", "col3": "teen spirit"}}, - {"topic": "test_topic", "key": 1, "value": {"col1": "the", "col2": "man who", "col3": "stole the world"}}, - {"topic": "test_topic", "key": 2, "value": {"col1": "smells", "col2": "like", "col3": "spring"}}, - {"topic": "test_topic", "key": 3, "value": {"col1": "smells", "col2": "like", "col3": " teen spirit "}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "teen spirit"}, "value": {"FOO": "Smellsliketeen spirit", "KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"G1": "The", "G2": "man who", "G3": "stole the world"}, "value": {"FOO": "Theman whostole the world", "KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "spring"}, "value": {"FOO": "Smellslikespring","KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": " teen spirit "}, "value": {"FOO": "Smellsliketeen spirit","KSQL_COL_0": 1}} - ] - }, - { - "name": "function group by column used in non-aggregate function in having", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', format='json');", - "CREATE TABLE OUTPUT AS SELECT INITCAP(COL1) AS G1, COL2 AS G2, trim(COL3) AS G3, COUNT(*) FROM input GROUP BY INITCAP(col1), col2, trim(col3) HAVING substring(trim(col3),1,4) = 'teen';" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"col1": "smells", "col2": "like", "col3": "teen spirit"}}, - {"topic": "test_topic", "key": 1, "value": {"col1": "the", "col2": "man who", "col3": "stole the world"}}, - {"topic": "test_topic", "key": 2, "value": {"col1": "smells", "col2": "like", "col3": "spring"}}, - {"topic": "test_topic", "key": 3, "value": {"col1": "smells", "col2": "like", "col3": " teen spirit "}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "teen spirit"}, "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "teen spirit"}, "value": {"KSQL_COL_0": 2}} - ] - }, - { - "name": "arithmetic in group by column used in non-aggregate function in select", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 INT, col2 INT) WITH (kafka_topic='test_topic', value_format='json');", - "CREATE TABLE OUTPUT AS SELECT col1+col2 AS G1, AS_VALUE(col1+col2), COUNT(*) FROM input GROUP BY col1+col2;" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"col1": 1, "col2": 1}}, - {"topic": "test_topic", "key": 1, "value": {"col1": 2, "col2": 2}}, - {"topic": "test_topic", "key": 2, "value": {"col1": 3, "col2": 3}}, - {"topic": "test_topic", "key": 3, "value": {"col1": 4, "col2": 4}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 2, "value": {"KSQL_COL_0": 2, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": 4, "value": {"KSQL_COL_0": 4, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": 6, "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": 8, "value": {"KSQL_COL_0": 8, "KSQL_COL_1": 1}} - ] - }, - { - "name": "expressions used in non-aggregate function in select whose children are not part of group-by", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 MAP, col2 MAP) WITH (kafka_topic='test_topic', value_format='json');", - "CREATE TABLE OUTPUT AS SELECT col1['foo']+col2['bar'] AS G1, AS_VALUE(col1['foo']+col2['bar']), COUNT(*) FROM input GROUP BY col1['foo']+col2['bar'];" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"col1": {"a" : 1}, "col2": {"b" : 1}}}, - {"topic": "test_topic", "key": 1, "value": {"col1": {"foo" : 1}, "col2": {"bar" : 1}}}, - {"topic": "test_topic", "key": 2, "value": {"col1": {"bar" : 1}, "col2": {"foo" : 1}}}, - {"topic": "test_topic", "key": 3, "value": {"col1": {"foo" : 1}, "col2": {"foo" : 1}}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 2, "value": {"KSQL_COL_0": 2, "KSQL_COL_1": 1}} - ] - }, - { - "name": "unknown function", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT DATA, WONT_FIND_ME(ID) FROM TEST GROUP BY DATA;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Can't find any functions with the name 'WONT_FIND_ME'" - } - }, - { - "name": "non-table udaf on table", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE INPUT (ID BIGINT PRIMARY KEY, F0 INT, F1 BIGINT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT ID, LATEST_BY_OFFSET(F0), MIN(F1), MAX(F1) FROM INPUT GROUP BY ID;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The aggregation functions LATEST_BY_OFFSET, MIN and MAX cannot be applied to a table source, only to a stream source." - } - }, - { - "name": "multiple expressions", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (f1 INT KEY, f2 INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT AS SELECT f2, f1, f2+f1, COUNT(*) FROM TEST GROUP BY f1, f2;" - ], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": {"f2": "2"}}, - {"topic": "test_topic", "key": 2, "value": {"f2": "4"}}, - {"topic": "test_topic", "key": 1, "value": {"f2": "2"}}, - {"topic": "test_topic", "key": 2, "value": {"f2": "4"}}, - {"topic": "test_topic", "key": 2, "value": {"f2": "1"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"F1": 1, "F2": 2}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": {"F1": 2, "F2": 4}, "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": {"F1": 1, "F2": 2}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 2}}, - {"topic": "OUTPUT", "key": {"F1": 2, "F2": 4}, "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 2}}, - {"topic": "OUTPUT", "key": {"F1": 2, "F2": 1}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "F1 INT KEY, F2 INT KEY, KSQL_COL_0 INT, KSQL_COL_1 BIGINT"} - ] - } - }, - { - "name": "multiple expressions with struct field and other expression", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (f1 INT KEY, f2 INT, address STRUCT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT AS SELECT address->town, f1, 2*f2+f1, COUNT(*), 2*f2 FROM TEST GROUP BY f1, address->town, 2*f2;" - ], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": {"f2": "2", "ADDRESS": {"STREET": "1st Street", "Town": "Oxford"}}}, - {"topic": "test_topic", "key": 2, "value": {"f2": "4", "ADDRESS": {"STREET": "1st Street", "Town": "London"}}}, - {"topic": "test_topic", "key": 1, "value": {"f2": "2", "ADDRESS": {"STREET": "1st Street", "Town": "Oxford"}}}, - {"topic": "test_topic", "key": 2, "value": {"f2": "4", "ADDRESS": {"STREET": "1st Street", "Town": "London"}}}, - {"topic": "test_topic", "key": 2, "value": {"f2": "1", "ADDRESS": {"STREET": "1st Street", "Town": "Oxford"}}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"F1": 1, "TOWN": "Oxford", "KSQL_COL_2": 4}, "value": {"KSQL_COL_0": 5, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": {"F1": 2, "TOWN": "London", "KSQL_COL_2": 8}, "value": {"KSQL_COL_0": 10, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": {"F1": 1, "TOWN": "Oxford", "KSQL_COL_2": 4}, "value": {"KSQL_COL_0": 5, "KSQL_COL_1": 2}}, - {"topic": "OUTPUT", "key": {"F1": 2, "TOWN": "London", "KSQL_COL_2": 8}, "value": {"KSQL_COL_0": 10, "KSQL_COL_1": 2}}, - {"topic": "OUTPUT", "key": {"F1": 2, "TOWN": "Oxford", "KSQL_COL_2": 2}, "value": {"KSQL_COL_0": 4, "KSQL_COL_1": 1}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "F1 INT KEY, TOWN STRING KEY, KSQL_COL_2 INT KEY, KSQL_COL_0 INT, KSQL_COL_1 BIGINT"} - ] - } - }, - { - "name": "multiple expressions - KAFKA key format", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (f1 INT KEY, f2 INT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT f2, f1, f2+f1, COUNT(*) FROM TEST GROUP BY f1, f2;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Key format does not support schema.\nformat: KAFKA\nschema: Persistence{columns=[`F1` INTEGER KEY, `F2` INTEGER KEY], features=[]}\nreason: The 'KAFKA' format only supports a single field. Got: [`F1` INTEGER KEY, `F2` INTEGER KEY]\nStatement: CREATE TABLE OUTPUT AS SELECT\n TEST.F2 F2,\n TEST.F1 F1,\n (TEST.F2 + TEST.F1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY TEST.F1, TEST.F2\nEMIT CHANGES" - } - }, - { - "name": "multiple expressions - windowed", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (f1 INT KEY, f2 INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT AS SELECT f2, f1, f2+f1, COUNT(*) FROM TEST WINDOW TUMBLING (SIZE 1 SECOND) GROUP BY f1, f2;" - ], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": {"f2": "2"}}, - {"topic": "test_topic", "key": 2, "value": {"f2": "4"}}, - {"topic": "test_topic", "key": 1, "value": {"f2": "2"}}, - {"topic": "test_topic", "key": 2, "value": {"f2": "4"}}, - {"topic": "test_topic", "key": 2, "value": {"f2": "1"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"F1": 1, "F2": 2}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": {"F1": 2, "F2": 4}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": {"F1": 1, "F2": 2}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 2}}, - {"topic": "OUTPUT", "key": {"F1": 2, "F2": 4}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 2}}, - {"topic": "OUTPUT", "key": {"F1": 2, "F2": 1}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "F1 INT KEY, F2 INT KEY, KSQL_COL_0 INT, KSQL_COL_1 BIGINT"} - ] - } - }, - { - "name": "multiple expressions - table aggregate", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE INPUT (ID INT PRIMARY KEY, VALUE INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT as SELECT 1 as k, value, count(1) AS ID FROM INPUT group by value, 1;" - ], - "inputs": [ - {"topic": "test_topic", "key": 10, "value": {"VALUE": 0}}, - {"topic": "test_topic", "key": 1666, "value": {"VALUE": 0}}, - {"topic": "test_topic", "key": 98, "value": {"VALUE": 0}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"VALUE": 0, "K": 1}, "value": {"ID": 1}}, - {"topic": "OUTPUT", "key": {"VALUE": 0, "K": 1}, "value": {"ID": 2}}, - {"topic": "OUTPUT", "key": {"VALUE": 0, "K": 1}, "value": {"ID": 3}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "VALUE INT KEY, K INT KEY, ID BIGINT"} - ] - } - }, - { - "name": "single expression - key in projection more than once", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (ID INT KEY, NAME STRING) WITH (kafka_topic='input',value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT NAME, NAME AS NAME2, COUNT(1) FROM INPUT GROUP BY NAME;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The projection contains a key column more than once: `NAME` and `NAME2`." - } - }, - { - "name": "single expression - key missing from projection - with other column of same name", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (ID INT KEY, NAME STRING) WITH (kafka_topic='input',value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT COUNT(1) AS NAME FROM INPUT GROUP BY NAME;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The query used to build `OUTPUT` must include the grouping expression NAME in its projection." - } - }, - { - "name": "single expression - key missing from projection", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (ID INT KEY, NAME STRING) WITH (kafka_topic='input',value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT COUNT(1) FROM INPUT GROUP BY NAME;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The query used to build `OUTPUT` must include the grouping expression NAME in its projection." - } - }, - { - "name": "multiple expressions - single key missing from projection", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (f1 INT KEY, f2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f2, COUNT(*) FROM TEST GROUP BY f1, f2;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The query used to build `OUTPUT` must include the grouping expression F1 in its projection." - } - }, - { - "name": "multiple expression - key in projection more than once", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (f1 INT KEY, f2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, f2, f1 AS F3, COUNT(*) FROM TEST GROUP BY f1, f2;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The projection contains a key column more than once: `F1` and `F3`." - } - }, - { - "name": "multiple expressions - all keys missing from projection", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (f1 INT KEY, f2 INT, f3 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT COUNT(*) FROM TEST GROUP BY f1, f2, f3;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The query used to build `OUTPUT` must include the grouping expressions F1, F2 and F3 in its projection." - } - }, - { - "name": "select * where all columns in group by", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (id INT KEY, id2 INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT AS SELECT *, COUNT() FROM TEST GROUP BY id, id2;" - ], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": {"ID2": 2}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"ID": 1, "ID2": 2}, "value": {"KSQL_COL_0": 1}} - ] - }, - { - "name": "select * where not all columns in group by", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (id INT KEY, id2 INT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT *, COUNT() FROM TEST GROUP BY id;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: ID2" - } - }, - { - "name": "with key alias that clashes with value alias", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT DATA AS COUNT, COUNT(*) AS COUNT FROM TEST GROUP BY DATA;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Duplicate value columns found in schema: `COUNT` BIGINT" - } - }, - { - "name": "map used in non-aggregate function in select when group by uses subscript", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 MAP) WITH (kafka_topic='test_topic', value_format='json');", - "CREATE TABLE OUTPUT AS SELECT col1['foo'], AS_VALUE(col1) AS foo, COUNT(*) FROM input GROUP BY col1['foo'];" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: AS_VALUE(COL1)\nEither add the column(s) to the GROUP BY or remove them from the SELECT." - } - }, - { - "name": "complex UDAF params", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (V0 INT KEY, V1 INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT AS SELECT V0, V1, SUM(V0 + V1) AS SUM FROM TEST GROUP BY V0, V1;" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"V1": 10}}, - {"topic": "test_topic", "key": 1, "value": {"V1": 20}}, - {"topic": "test_topic", "key": 0, "value": {"V1": 10}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"V0": 0, "V1": 10}, "value": {"SUM": 10}}, - {"topic": "OUTPUT", "key": {"V0": 1, "V1": 20}, "value": {"SUM": 21}}, - {"topic": "OUTPUT", "key": {"V0": 0, "V1": 10}, "value": {"SUM": 20}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "V0 INTEGER KEY, V1 INTEGER KEY, SUM INT"} - ] - } - }, - { - "name": "complex UDAF params matching GROUP BY", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (V0 INT KEY, V1 INT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT (V0 + V1) AS NEW_KEY, SUM(V0 + V1) AS SUM FROM TEST GROUP BY V0 + V1;" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"V1": 10}}, - {"topic": "test_topic", "key": 1, "value": {"V1": 20}}, - {"topic": "test_topic", "key": 0, "value": {"V1": 10}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 10, "value": {"SUM": 10}}, - {"topic": "OUTPUT", "key": 21, "value": {"SUM": 21}}, - {"topic": "OUTPUT", "key": 10, "value": {"SUM": 20}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "NEW_KEY INT KEY, SUM INT"} - ] - } - }, - { - "name": "complex UDAF params matching HAVING", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (V0 INT KEY, V1 INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT AS SELECT SUM(V0 + V1) AS SUM, V0, V1 FROM TEST GROUP BY V0, V1 HAVING V0 + V1 <= 20;" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"V1": 10}}, - {"topic": "test_topic", "key": 1, "value": {"V1": 20}}, - {"topic": "test_topic", "key": 0, "value": {"V1": 10}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"V0": 0, "V1": 10}, "value": {"SUM": 10}}, - {"topic": "OUTPUT", "key": {"V0": 0, "V1": 10}, "value": {"SUM": 20}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "V0 INTEGER KEY, V1 INTEGER KEY, SUM INT"} - ] - } - }, - { - "name": "single expression with nulls", - "comment": "bad_udf return every other invocation - tables need a PRIMARY key so null keys are dropped", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT bad_udf(DATA), COUNT(*) FROM TEST GROUP BY bad_udf(DATA);" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"data": "d1"}}, - {"topic": "test_topic", "key": 1, "value": {"data": "d2"}}, - {"topic": "test_topic", "key": 2, "value": {"data": "d1"}}, - {"topic": "test_topic", "key": 3, "value": {"data": "d2"}}, - {"topic": "test_topic", "key": 4, "value": {"data": "d1"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d2", "value": {"KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": "d2", "value": {"KSQL_COL_1": 2}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_0 STRING KEY, KSQL_COL_1 BIGINT"} - ] - } - }, - { - "name": "complex expressions", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT COUNT(*) FROM TEST GROUP BY DATA AND ID;" - ], - "expectedException": { - "type": "io.confluent.ksql.parser.exception.ParseFailedException", - "message": "mismatched input 'AND' expecting ';'" - } - }, - { - "name": "multiple expressions with nulls", - "comment": "bad_udf returns null every other invocation - tables need a PRIMARY key so null keys are dropped", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT bad_udf(DATA), COUNT(*) FROM TEST GROUP BY bad_udf(DATA);" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"data": "d1"}}, - {"topic": "test_topic", "key": 1, "value": {"data": "d2"}}, - {"topic": "test_topic", "key": 2, "value": {"data": "d1"}}, - {"topic": "test_topic", "key": 3, "value": {"data": "d2"}}, - {"topic": "test_topic", "key": 4, "value": {"data": "d1"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d2", "value": {"KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": "d2", "value": {"KSQL_COL_1": 2}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_0 STRING KEY, KSQL_COL_1 BIGINT"} - ] - } - }, - { - "name": "field (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, ignored VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT K, COUNT(*) FROM TEST GROUP BY K;" - ], - "inputs": [ - {"topic": "test_topic", "key": "d1", "value": "-"}, - {"topic": "test_topic", "key": "d2", "value": "-"}, - {"topic": "test_topic", "key": "d1", "value": "-"}, - {"topic": "test_topic", "key": "d2", "value": "-"}, - {"topic": "test_topic", "key": "d1", "value": "-"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d1", "value": "1"}, - {"topic": "OUTPUT", "key": "d2", "value": "1"}, - {"topic": "OUTPUT", "key": "d1", "value": "2"}, - {"topic": "OUTPUT", "key": "d2", "value": "2"}, - {"topic": "OUTPUT", "key": "d1", "value": "3"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "K STRING KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "field (stream->table) - KAFKA", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='KAFKA');", - "CREATE TABLE OUTPUT WITH(value_format='DELIMITED') AS SELECT DATA, COUNT(*) FROM TEST GROUP BY DATA;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Source(s) TEST are using the 'KAFKA' value format. This format does not yet support GROUP BY." - } - }, - { - "name": "field (stream->table) - format", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, ignored VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", - "CREATE TABLE OUTPUT AS SELECT K, COUNT(*) FROM TEST GROUP BY K;" - ], - "format": ["AVRO", "JSON", "PROTOBUF"], - "inputs": [ - {"topic": "test_topic", "key": "d1", "value": {}, "timestamp": 1}, - {"topic": "test_topic", "key": "d2", "value": {}, "timestamp": 2}, - {"topic": "test_topic", "key": "d1", "value": {}, "timestamp": 3}, - {"topic": "test_topic", "key": "d2", "value": {}, "timestamp": 4}, - {"topic": "test_topic", "key": "d1", "value": {}, "timestamp": 5} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0":1}, "timestamp": 1}, - {"topic": "OUTPUT", "key": "d2", "value": {"KSQL_COL_0":1}, "timestamp": 2}, - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0":2}, "timestamp": 3}, - {"topic": "OUTPUT", "key": "d2", "value": {"KSQL_COL_0":2}, "timestamp": 4}, - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0":3}, "timestamp": 5} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "K STRING KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "int field (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, ID INT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT ID, COUNT(*) FROM TEST GROUP BY ID;" - ], - "inputs": [ - {"topic": "test_topic", "key": "a", "value": {"ID": 1}, "timestamp": 1}, - {"topic": "test_topic", "key": "b", "value": {"ID": 2}, "timestamp": 2}, - {"topic": "test_topic", "key": "c", "value": {"ID": 1}, "timestamp": 3}, - {"topic": "test_topic", "key": "d", "value": {"ID": 2}, "timestamp": 4}, - {"topic": "test_topic", "key": "e", "value": {"ID": 1}, "timestamp": 5} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 1, "value": {"KSQL_COL_0":1}, "timestamp": 1}, - {"topic": "OUTPUT", "key": 2, "value": {"KSQL_COL_0":1}, "timestamp": 2}, - {"topic": "OUTPUT", "key": 1, "value": {"KSQL_COL_0":2}, "timestamp": 3}, - {"topic": "OUTPUT", "key": 2, "value": {"KSQL_COL_0":2}, "timestamp": 4}, - {"topic": "OUTPUT", "key": 1, "value": {"KSQL_COL_0":3}, "timestamp": 5} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "ID INT KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "fields (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" - ], - "inputs": [ - {"topic": "test_topic", "key": "1", "value": "1,a"}, - {"topic": "test_topic", "key": "2", "value": "2,b"}, - {"topic": "test_topic", "key": "1", "value": "1,a"}, - {"topic": "test_topic", "key": "2", "value": "2,b"}, - {"topic": "test_topic", "key": "3", "value": "3,a"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "a,1", "value": "1"}, - {"topic": "OUTPUT", "key": "b,2", "value": "1"}, - {"topic": "OUTPUT", "key": "a,1", "value": "2"}, - {"topic": "OUTPUT", "key": "b,2", "value": "2"}, - {"topic": "OUTPUT", "key": "a,3", "value": "1"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "F2 STRING KEY, F1 INTEGER KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "fields used in expression", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 INT) WITH (kafka_topic='test_topic', format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT F1, F2, f1 / f2, COUNT(*) FROM TEST GROUP BY f1, f2;" - ], - "inputs": [ - {"topic": "test_topic", "value": "4,2"}, - {"topic": "test_topic", "value": "9,3"}, - {"topic": "test_topic", "value": "9,3"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "4,2", "value": "2,1"}, - {"topic": "OUTPUT", "key": "9,3", "value": "3,1"}, - {"topic": "OUTPUT", "key": "9,3", "value": "3,2"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "F1 INTEGER KEY, F2 INTEGER KEY, KSQL_COL_0 INTEGER, KSQL_COL_1 BIGINT"} - ] - } - }, - { - "name": "fields (stream->table) - format", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', key_format='JSON', value_format='{FORMAT}');", - "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" - ], - "format": ["AVRO", "JSON", "PROTOBUF"], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": {"F1": 1, "F2": "a"}}, - {"topic": "test_topic", "key": 2, "value": {"F1": 2, "F2": "b"}}, - {"topic": "test_topic", "key": 1, "value": {"F1": 1, "F2": "a"}}, - {"topic": "test_topic", "key": 2, "value": {"F1": 2, "F2": "b"}}, - {"topic": "test_topic", "key": 3, "value": {"F1": 3, "F2": "a"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"F2": "a", "F1": 1}, "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"F2": "b", "F1": 2}, "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"F2": "a", "F1": 1}, "value": {"KSQL_COL_0": 2}}, - {"topic": "OUTPUT", "key": {"F2": "b", "F1": 2}, "value": {"KSQL_COL_0": 2}}, - {"topic": "OUTPUT", "key": {"F2": "a", "F1": 3}, "value": {"KSQL_COL_0": 1}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "F2 STRING KEY, F1 INTEGER KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "with groupings (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR, f3 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f3, (f2, f1);" - ], - "expectedException": { - "type": "io.confluent.ksql.parser.exception.ParseFailedException", - "message": "line 2:74: missing ')' at ','" - } - }, - { - "name": "duplicate expressions", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, data STRING) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT COUNT(*) FROM TEST GROUP BY DATA, TEST.DATA;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Duplicate GROUP BY expression: TEST.DATA" - } - }, - { - "name": "with single grouping set (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR, f3 INT) WITH (kafka_topic='test_topic', format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, f2, f3, COUNT(*) FROM TEST GROUP BY (f3, f2, f1);" - ], - "inputs": [ - {"topic": "test_topic", "key": "1", "value": "1,a,-1"}, - {"topic": "test_topic", "key": "2", "value": "2,b,-2"}, - {"topic": "test_topic", "key": "1", "value": "1,a,-1"}, - {"topic": "test_topic", "key": "2", "value": "2,b,-2"}, - {"topic": "test_topic", "key": "3", "value": "3,a,-3"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "\"-1\",a,1", "value": "1"}, - {"topic": "OUTPUT", "key": "\"-2\",b,2", "value": "1"}, - {"topic": "OUTPUT", "key": "\"-1\",a,1", "value": "2"}, - {"topic": "OUTPUT", "key": "\"-2\",b,2", "value": "2"}, - {"topic": "OUTPUT", "key": "\"-3\",a,3", "value": "1"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "F3 INTEGER KEY, F2 STRING KEY, F1 INTEGER KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "fields (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" - ], - "inputs": [ - {"topic": "test_topic", "key": "1", "value": "1,a"}, - {"topic": "test_topic", "key": "2", "value": "2,b"}, - {"topic": "test_topic", "key": "1", "value": "1,b"}, - {"topic": "test_topic", "key": "2", "value": null}, - {"topic": "test_topic", "key": "1", "value": "1,a"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "a,1", "value": "1"}, - {"topic": "OUTPUT", "key": "b,2", "value": "1"}, - {"topic": "OUTPUT", "key": "a,1", "value": "0"}, - {"topic": "OUTPUT", "key": "b,1", "value": "1"}, - {"topic": "OUTPUT", "key": "b,2", "value": "0"}, - {"topic": "OUTPUT", "key": "b,1", "value": "0"}, - {"topic": "OUTPUT", "key": "a,1", "value": "1"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "F2 STRING KEY, F1 INTEGER KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "fields - copied into value (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, f2, AS_VALUE(f1) AS F3, AS_VALUE(F2) AS F4, COUNT(*) FROM TEST GROUP BY f2, f1;" - ], - "inputs": [ - {"topic": "test_topic", "key": "1", "value": "1,a"}, - {"topic": "test_topic", "key": "2", "value": "2,b"}, - {"topic": "test_topic", "key": "1", "value": "1,b"}, - {"topic": "test_topic", "key": "2", "value": null}, - {"topic": "test_topic", "key": "1", "value": "1,a"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "a,1", "value": "1,a,1"}, - {"topic": "OUTPUT", "key": "b,2", "value": "2,b,1"}, - {"topic": "OUTPUT", "key": "a,1", "value": "1,a,0"}, - {"topic": "OUTPUT", "key": "b,1", "value": "1,b,1"}, - {"topic": "OUTPUT", "key": "b,2", "value": "2,b,0"}, - {"topic": "OUTPUT", "key": "b,1", "value": "1,b,0"}, - {"topic": "OUTPUT", "key": "a,1", "value": "1,a,1"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "F2 STRING KEY, F1 INTEGER KEY, F3 INT, F4 STRING, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "fields (table->table) - format", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', key_format='JSON', value_format='{FORMAT}');", - "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" - ], - "format": ["AVRO", "JSON", "PROTOBUF"], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": {"F1": 1, "F2": "a"}}, - {"topic": "test_topic", "key": 2, "value": {"F1": 2, "F2": "b"}}, - {"topic": "test_topic", "key": 1, "value": {"F1": 1, "F2": "b"}}, - {"topic": "test_topic", "key": 2, "value": null}, - {"topic": "test_topic", "key": 1, "value": {"F1": 1, "F2": "a"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"F2": "a", "F1": 1}, "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"F2": "b", "F1": 2}, "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"F2": "a", "F1": 1}, "value": {"KSQL_COL_0": 0}}, - {"topic": "OUTPUT", "key": {"F2": "b", "F1": 1}, "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": {"F2": "b", "F1": 2}, "value": {"KSQL_COL_0": 0}}, - {"topic": "OUTPUT", "key": {"F2": "b", "F1": 1}, "value": {"KSQL_COL_0": 0}}, - {"topic": "OUTPUT", "key": {"F2": "a", "F1": 1}, "value": {"KSQL_COL_0": 1}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "F2 STRING KEY, F1 INTEGER KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "field with re-key (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) FROM TEST GROUP BY DATA;" - ], - "inputs": [ - {"topic": "test_topic", "value": "d1"}, - {"topic": "test_topic", "value": "d2"}, - {"topic": "test_topic", "value": "d1"}, - {"topic": "test_topic", "value": "d2"}, - {"topic": "test_topic", "value": "d1"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d1", "value": "1"}, - {"topic": "OUTPUT", "key": "d2", "value": "1"}, - {"topic": "OUTPUT", "key": "d1", "value": "2"}, - {"topic": "OUTPUT", "key": "d2", "value": "2"}, - {"topic": "OUTPUT", "key": "d1", "value": "3"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "DATA STRING KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "double field with re-key (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, data double) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) FROM TEST GROUP BY DATA;" - ], - "inputs": [ - {"topic": "test_topic", "value": "0.1"}, - {"topic": "test_topic", "value": "0.2"}, - {"topic": "test_topic", "value": "0.1"}, - {"topic": "test_topic", "value": "0.2"}, - {"topic": "test_topic", "value": "0.1"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 0.1, "value": "1"}, - {"topic": "OUTPUT", "key": 0.2, "value": "1"}, - {"topic": "OUTPUT", "key": 0.1, "value": "2"}, - {"topic": "OUTPUT", "key": 0.2, "value": "2"}, - {"topic": "OUTPUT", "key": 0.1, "value": "3"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "DATA DOUBLE KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "field with re-key (stream->table) - format", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", - "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*) FROM TEST GROUP BY DATA;" - ], - "format": ["AVRO", "JSON", "PROTOBUF"], - "inputs": [ - {"topic": "test_topic", "value": {"DATA": "d1"}}, - {"topic": "test_topic", "value": {"DATA": "d2"}}, - {"topic": "test_topic", "value": {"DATA": "d1"}}, - {"topic": "test_topic", "value": {"DATA": "d2"}}, - {"topic": "test_topic", "value": {"DATA": "d1"}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0":1}}, - {"topic": "OUTPUT", "key": "d2", "value": {"KSQL_COL_0":1}}, - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0":2}}, - {"topic": "OUTPUT", "key": "d2", "value": {"KSQL_COL_0":2}}, - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0":3}} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "DATA STRING KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "field with re-key (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT region, COUNT(*) FROM TEST GROUP BY region;" - ], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": "1,r0"}, - {"topic": "test_topic", "key": 2, "value": "2,r1"}, - {"topic": "test_topic", "key": 3, "value": "3,r0"}, - {"topic": "test_topic", "key": 1, "value": null}, - {"topic": "test_topic", "key": 2, "value": "2,r0"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "r0", "value": "1"}, - {"topic": "OUTPUT", "key": "r1", "value": "1"}, - {"topic": "OUTPUT", "key": "r0", "value": "2"}, - {"topic": "OUTPUT", "key": "r0", "value": "1"}, - {"topic": "OUTPUT", "key": "r1", "value": "0"}, - {"topic": "OUTPUT", "key": "r0", "value": "2"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "REGION STRING KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "with aggregate arithmetic (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(*)*2 FROM TEST GROUP BY DATA;" - ], - "inputs": [ - {"topic": "test_topic", "value": "d1"}, - {"topic": "test_topic", "value": "d2"}, - {"topic": "test_topic", "value": "d1"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d1", "value": "2"}, - {"topic": "OUTPUT", "key": "d2", "value": "2"}, - {"topic": "OUTPUT", "key": "d1", "value": "4"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "DATA STRING KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "with aggregate arithmetic (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT region, COUNT(*) * 2 FROM TEST GROUP BY region;" - ], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": "1,r0"}, - {"topic": "test_topic", "key": 2, "value": "2,r1"}, - {"topic": "test_topic", "key": 3, "value": "3,r0"}, - {"topic": "test_topic", "key": 1, "value": null}, - {"topic": "test_topic", "key": 2, "value": "2,r0"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "r0", "value": "2"}, - {"topic": "OUTPUT", "key": "r1", "value": "2"}, - {"topic": "OUTPUT", "key": "r0", "value": "4"}, - {"topic": "OUTPUT", "key": "r0", "value": "2"}, - {"topic": "OUTPUT", "key": "r1", "value": "0"}, - {"topic": "OUTPUT", "key": "r0", "value": "4"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "REGION STRING KEY, KSQL_COL_0 BIGINT"} - ] - } - }, - { - "name": "with aggregate arithmetic involving source field (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, ITEM INT, COST INT) WITH (kafka_topic='test_topic', format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT ITEM, COST, COST * COUNT() FROM TEST GROUP BY ITEM, COST;" - ], - "inputs": [ - {"topic": "test_topic", "value": "1,10"}, - {"topic": "test_topic", "value": "1,20"}, - {"topic": "test_topic", "value": "2,30"}, - {"topic": "test_topic", "value": "1,10"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "1,10", "value": "10"}, - {"topic": "OUTPUT", "key": "1,20", "value": "20"}, - {"topic": "OUTPUT", "key": "2,30", "value": "30"}, - {"topic": "OUTPUT", "key": "1,10", "value": "20"} - ] - }, - { - "name": "with aggregate arithmetic involving source field (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, f0 INT, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f0, f0 * SUM(f1) FROM TEST GROUP BY f0;" - ], - "inputs": [ - {"topic": "test_topic", "key": 2, "value": "2,10"}, - {"topic": "test_topic", "key": 2, "value": "2,20"}, - {"topic": "test_topic", "key": 2, "value": "2,30"}, - {"topic": "test_topic", "key": 2, "value": null} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 2, "value": "20"}, - {"topic": "OUTPUT", "key": 2, "value": "0"}, - {"topic": "OUTPUT", "key": 2, "value": "40"}, - {"topic": "OUTPUT", "key": 2, "value": "0"}, - {"topic": "OUTPUT", "key": 2, "value": "60"}, - {"topic": "OUTPUT", "key": 2, "value": "0"} - ] - }, - { - "name": "with aggregate arithmetic involving source field not in group by (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, f0 INT, f1 INT, f2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1 * SUM(f2) FROM TEST GROUP BY f0;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: (F1 * SUM(F2))\nEither add the column(s) to the GROUP BY or remove them from the SELECT." - } - }, - { - "name": "function (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT SUBSTRING(source, 0, 2), COUNT(*) FROM TEST GROUP BY SUBSTRING(source, 0, 2);" - ], - "inputs": [ - {"topic": "test_topic", "value": "some string"}, - {"topic": "test_topic", "value": "another string"}, - {"topic": "test_topic", "value": "some string again"}, - {"topic": "test_topic", "value": "another string again"}, - {"topic": "test_topic", "value": "some other string"}, - {"topic": "test_topic", "value": "the final string"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "so", "value": "1"}, - {"topic": "OUTPUT", "key": "an", "value": "1"}, - {"topic": "OUTPUT", "key": "so", "value": "2"}, - {"topic": "OUTPUT", "key": "an", "value": "2"}, - {"topic": "OUTPUT", "key": "so", "value": "3"}, - {"topic": "OUTPUT", "key": "th", "value": "1"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_0 STRING KEY, KSQL_COL_1 BIGINT"} - ] - } - }, - { - "name": "function (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT SUBSTRING(region, 7, 2), COUNT(*) FROM TEST GROUP BY SUBSTRING(region, 7, 2);" - ], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": "1,prefixr0"}, - {"topic": "test_topic", "key": 2, "value": "2,prefixr1"}, - {"topic": "test_topic", "key": 3, "value": "3,prefixr0"}, - {"topic": "test_topic", "key": 1, "value": null}, - {"topic": "test_topic", "key": 2, "value": "2,prefixr0"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "r0", "value": "1"}, - {"topic": "OUTPUT", "key": "r1", "value": "1"}, - {"topic": "OUTPUT", "key": "r0", "value": "2"}, - {"topic": "OUTPUT", "key": "r0", "value": "1"}, - {"topic": "OUTPUT", "key": "r1", "value": "0"}, - {"topic": "OUTPUT", "key": "r0", "value": "2"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_0 STRING KEY, KSQL_COL_1 BIGINT"} - ] - } - }, - { - "name": "int function (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT LEN(region), COUNT(*) FROM TEST GROUP BY LEN(region);" - ], - "inputs": [ - {"topic": "test_topic", "key": "1", "value": "usa"}, - {"topic": "test_topic", "key": "2", "value": "eu"}, - {"topic": "test_topic", "key": "3", "value": "usa"}, - {"topic": "test_topic", "key": "1", "value": null}, - {"topic": "test_topic", "key": "2", "value": "usa"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 3, "value": "1"}, - {"topic": "OUTPUT", "key": 2, "value": "1"}, - {"topic": "OUTPUT", "key": 3, "value": "2"}, - {"topic": "OUTPUT", "key": 3, "value": "1"}, - {"topic": "OUTPUT", "key": 2, "value": "0"}, - {"topic": "OUTPUT", "key": 3, "value": "2"} - ], - "post": { - "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_0 INT KEY, KSQL_COL_1 BIGINT"} - ] - } - }, - { - "name": "function with select field that is a subset of group by (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT SUBSTRING(source, 0, 1) AS Thing, COUNT(*) FROM TEST GROUP BY SUBSTRING(source, 0, 2);" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: SUBSTRING(SOURCE, 0, 1)" - } - }, - { - "name": "function with select field that is a subset of group by (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT SUBSTRING(region, 7, 1), COUNT(*) FROM TEST GROUP BY SUBSTRING(region, 7, 2);" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: SUBSTRING(REGION, 7, 1)" - } - }, - { - "name": "function with select field that is a superset of group by (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT SUBSTRING(source, 0, 3), COUNT(*) FROM TEST GROUP BY SUBSTRING(source, 0, 2);" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: SUBSTRING(SOURCE, 0, 3)" - } - }, - { - "name": "function with select field that is a superset of group by (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT SUBSTRING(region, 7, 3), COUNT(*) FROM TEST GROUP BY SUBSTRING(region, 7, 2);" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: SUBSTRING(REGION, 7, 3)" - } - }, - { - "name": "function with having field that is a subset of group by (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT SUBSTRING(source, 0, 2) AS Thing, COUNT(*) FROM TEST GROUP BY SUBSTRING(source, 0, 2) HAVING LEN(source) < 2;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate HAVING expression not part of GROUP BY: SOURCE" - } - }, - { - "name": "json field (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (data STRUCT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT data->field AS FIELD, COUNT(*) AS COUNT FROM TEST GROUP BY data->field;" - ], - "inputs": [ - {"topic": "test_topic", "value": {"data": {"field": "Something"}}}, - {"topic": "test_topic", "value": {"data": {"field": "Something Else"}}}, - {"topic": "test_topic", "value": {"data": {}}}, - {"topic": "test_topic", "value": {"data": {"field": "Something"}}}, - {"topic": "test_topic", "value": {"data": {}}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "Something", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "Something Else", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "Something", "value": {"COUNT": 2}} - ] - }, - { - "name": "int json field (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (data STRUCT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT data->field, COUNT(*) AS COUNT FROM TEST GROUP BY data->field;" - ], - "inputs": [ - {"topic": "test_topic", "value": {"data": {"field": 1}}}, - {"topic": "test_topic", "value": {"data": {"field": 2}}}, - {"topic": "test_topic", "value": {"data": {}}}, - {"topic": "test_topic", "value": {"data": {"field": 1}}}, - {"topic": "test_topic", "value": {"data": {}}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 1, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": 1, "value": {"COUNT": 2}} - ] - }, - { - "name": "key (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, ignored VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT K, COUNT(*) FROM TEST GROUP BY K;" - ], - "inputs": [ - {"topic": "test_topic", "key": "1", "value": "-"}, - {"topic": "test_topic", "key": "2", "value": "-"}, - {"topic": "test_topic", "key": "1", "value": "-"}, - {"topic": "test_topic", "key": "2", "value": "-"}, - {"topic": "test_topic", "key": "1", "value": "-"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "1", "value": "1"}, - {"topic": "OUTPUT", "key": "2", "value": "1"}, - {"topic": "OUTPUT", "key": "1", "value": "2"}, - {"topic": "OUTPUT", "key": "2", "value": "2"}, - {"topic": "OUTPUT", "key": "1", "value": "3"} - ], - "post": { - "topics": { - "blacklist": ".*-repartition" - } - } - }, - { - "name": "ROWTIME (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, ignored VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT ROWTIME AS RT, COUNT(*) FROM TEST GROUP BY ROWTIME;" - ], - "inputs": [{"topic": "test_topic", "value": "-", "timestamp": 10}], - "outputs": [{"topic": "OUTPUT", "key": 10, "value": "1", "timestamp": 10}] - }, - { - "name": "constant (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, ignored VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT 1, COUNT(*) FROM TEST GROUP BY 1;" - ], - "inputs": [ - {"topic": "test_topic", "value": "-"}, - {"topic": "test_topic", "value": "-"}, - {"topic": "test_topic", "value": "-"}, - {"topic": "test_topic", "value": "-"}, - {"topic": "test_topic", "value": "-"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 1, "value": "1"}, - {"topic": "OUTPUT", "key": 1, "value": "2"}, - {"topic": "OUTPUT", "key": 1, "value": "3"}, - {"topic": "OUTPUT", "key": 1, "value": "4"}, - {"topic": "OUTPUT", "key": 1, "value": "5"} - ] - }, - { - "name": "constant (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT 1, COUNT(*) FROM TEST GROUP BY 1;" - ], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": "1,r0"}, - {"topic": "test_topic", "key": 2, "value": "2,r1"}, - {"topic": "test_topic", "key": 3, "value": "3,r0"}, - {"topic": "test_topic", "key": 1, "value": null}, - {"topic": "test_topic", "key": 2, "value": "2,r0"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 1, "value": "1"}, - {"topic": "OUTPUT", "key": 1, "value": "2"}, - {"topic": "OUTPUT", "key": 1, "value": "3"}, - {"topic": "OUTPUT", "key": 1, "value": "2"}, - {"topic": "OUTPUT", "key": 1, "value": "1"}, - {"topic": "OUTPUT", "key": 1, "value": "2"} - ] - }, - { - "name": "field with field used in function in projection (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, f1 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, SUBSTRING(f1, 0, 1), COUNT(*) FROM TEST GROUP BY f1;" - ], - "inputs": [ - {"topic": "test_topic", "value": "one"}, - {"topic": "test_topic", "value": "two"}, - {"topic": "test_topic", "value": "three"}, - {"topic": "test_topic", "value": "one"}, - {"topic": "test_topic", "value": "five"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "one", "value": "o,1"}, - {"topic": "OUTPUT", "key": "two", "value": "t,1"}, - {"topic": "OUTPUT", "key": "three", "value": "t,1"}, - {"topic": "OUTPUT", "key": "one", "value": "o,2"}, - {"topic": "OUTPUT", "key": "five", "value": "f,1"} - ] - }, - { - "name": "field with field used in function in projection (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, user INT, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT region, SUBSTRING(region, 2, 1), COUNT(*) FROM TEST GROUP BY region;" - ], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": "1,r0"}, - {"topic": "test_topic", "key": 2, "value": "2,r1"}, - {"topic": "test_topic", "key": 3, "value": "3,r0"}, - {"topic": "test_topic", "key": 1, "value": null}, - {"topic": "test_topic", "key": 2, "value": "2,r0"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "r0", "value": "0,1"}, - {"topic": "OUTPUT", "key": "r1", "value": "1,1"}, - {"topic": "OUTPUT", "key": "r0", "value": "0,2"}, - {"topic": "OUTPUT", "key": "r0", "value": "0,1"}, - {"topic": "OUTPUT", "key": "r1", "value": "1,0"}, - {"topic": "OUTPUT", "key": "r0", "value": "0,2"} - ] - }, - { - "name": "string concat using + op (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, f1 VARCHAR, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f2 + f1, COUNT(*) FROM TEST GROUP BY f2 + f1;" - ], - "inputs": [ - {"topic": "test_topic", "value": "1,a"}, - {"topic": "test_topic", "value": "2,b"}, - {"topic": "test_topic", "value": "1,a"}, - {"topic": "test_topic", "value": "2,b"}, - {"topic": "test_topic", "value": "3,a"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "a1", "value": "1"}, - {"topic": "OUTPUT", "key": "b2", "value": "1"}, - {"topic": "OUTPUT", "key": "a1", "value": "2"}, - {"topic": "OUTPUT", "key": "b2", "value": "2"}, - {"topic": "OUTPUT", "key": "a3", "value": "1"} - ] - }, - { - "name": "string concat using + op (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, user INT, subregion VARCHAR, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT region + subregion, COUNT(*) FROM TEST GROUP BY region + subregion;" - ], - "inputs": [ - {"topic": "test_topic", "key": "1", "value": "1,a,r0"}, - {"topic": "test_topic", "key": "2", "value": "2,a,r1"}, - {"topic": "test_topic", "key": "3", "value": "3,a,r0"}, - {"topic": "test_topic", "key": "4", "value": "4,b,r0"}, - {"topic": "test_topic", "key": "1", "value": null}, - {"topic": "test_topic", "key": "2", "value": "2,a,r0"}, - {"topic": "test_topic", "key": "2", "value": "2,b,r1"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "r0a", "value": "1"}, - {"topic": "OUTPUT", "key": "r1a", "value": "1"}, - {"topic": "OUTPUT", "key": "r0a", "value": "2"}, - {"topic": "OUTPUT", "key": "r0b", "value": "1"}, - {"topic": "OUTPUT", "key": "r0a", "value": "1"}, - {"topic": "OUTPUT", "key": "r1a", "value": "0"}, - {"topic": "OUTPUT", "key": "r0a", "value": "2"}, - {"topic": "OUTPUT", "key": "r0a", "value": "1"}, - {"topic": "OUTPUT", "key": "r1b", "value": "1"} - ] - }, - { - "name": "string concat using + op with projection field in wrong order (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, f1 VARCHAR, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1 + f2, COUNT(*) FROM TEST GROUP BY f2 + f1;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: (F1 + F2)" - } - }, - { - "name": "string concat using + op with projection field in wrong order (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, user INT, subregion VARCHAR, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT subregion + region, COUNT(*) FROM TEST GROUP BY region + subregion;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: (SUBREGION + REGION)" - } - }, - { - "name": "string concat with separate fields in projection (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, f1 VARCHAR, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2 + f1;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: F1, F2" - } - }, - { - "name": "string concat with separate fields in projection (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, user INT, subregion VARCHAR, region VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT subregion, region, COUNT(*) FROM TEST GROUP BY region + subregion;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: SUBREGION, REGION" - } - }, - { - "name": "arithmetic binary expression with projection in-order & non-commutative group by (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f2 - f1, COUNT(*) FROM TEST GROUP BY f2 - f1;" - ], - "inputs": [ - {"topic": "test_topic", "value": "1,2"}, - {"topic": "test_topic", "value": "2,3"}, - {"topic": "test_topic", "value": "2,4"}, - {"topic": "test_topic", "value": "6,8"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 1, "value": "1"}, - {"topic": "OUTPUT", "key": 1, "value": "2"}, - {"topic": "OUTPUT", "key": 2, "value": "1"}, - {"topic": "OUTPUT", "key": 2, "value": "2"} - ] - }, - { - "name": "arithmetic binary expression with projection in-order & non-commutative group by (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, f0 INT, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f0 - f1, COUNT(*) FROM TEST GROUP BY f0 - f1;" - ], - "inputs": [ - {"topic": "test_topic", "key": "1", "value": "1,0"}, - {"topic": "test_topic", "key": "2", "value": "2,1"}, - {"topic": "test_topic", "key": "3", "value": "3,1"}, - {"topic": "test_topic", "key": "1", "value": null}, - {"topic": "test_topic", "key": "2", "value": "4,2"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 1, "value": "1"}, - {"topic": "OUTPUT", "key": 1, "value": "2"}, - {"topic": "OUTPUT", "key": 2, "value": "1"}, - {"topic": "OUTPUT", "key": 1, "value": "1"}, - {"topic": "OUTPUT", "key": 1, "value": "0"}, - {"topic": "OUTPUT", "key": 2, "value": "2"} - ] - }, - { - "name": "arithmetic binary expression with projection out-of-order & non-commutative group by (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1 - f2, COUNT(*) FROM TEST GROUP BY f2 - f1;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: (F1 - F2)" - } - }, - { - "name": "arithmetic binary expression with projection out-of-order & non-commutative group by (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, f0 INT, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1 - f0, COUNT(*) FROM TEST GROUP BY f0 - f1;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: (F1 - F0)" - } - }, - { - "name": "with having expression (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, COUNT(*) FROM TEST GROUP BY f1 HAVING SUM(f1) > 1;" - ], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": "1"}, - {"topic": "test_topic", "key": 2, "value": "2"}, - {"topic": "test_topic", "key": 1, "value": "1"}, - {"topic": "test_topic", "key": 2, "value": "2"}, - {"topic": "test_topic", "key": 3, "value": "3"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 2, "value": "1"}, - {"topic": "OUTPUT", "key": 1, "value": "2"}, - {"topic": "OUTPUT", "key": 2, "value": "2"}, - {"topic": "OUTPUT", "key": 3, "value": "1"} - ] - }, - { - "name": "with having expression (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, f0 INT, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, SUM(f0) FROM TEST GROUP BY f1 HAVING COUNT(f1) > 0;" - ], - "inputs": [ - {"topic": "test_topic", "key": "1", "value": "1,0", "timestamp": 1}, - {"topic": "test_topic", "key": "2", "value": "2,1", "timestamp": 2}, - {"topic": "test_topic", "key": "1", "value": null, "timestamp": 3}, - {"topic": "test_topic", "key": "3", "value": "3,0", "timestamp": 4}, - {"topic": "test_topic", "key": "2", "value": "2,0", "timestamp": 5} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 0, "value": "1", "timestamp": 1}, - {"topic": "OUTPUT", "key": 1, "value": "2", "timestamp": 2}, - {"topic": "OUTPUT", "key": 0, "value": null, "timestamp": 3}, - {"topic": "OUTPUT", "key": 0, "value": "3", "timestamp": 4}, - {"topic": "OUTPUT", "key": 1, "value": null, "timestamp": 5}, - {"topic": "OUTPUT", "key": 0, "value": "5", "timestamp": 5} - ] - }, - { - "name": "with multiple having expressions (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, COUNT(f1) FROM TEST GROUP BY f1 HAVING COUNT(f1) > 1 AND f1=1;" - ], - "inputs": [ - {"topic": "test_topic", "key": "0", "value": "1,a"}, - {"topic": "test_topic", "key": "0", "value": "2,b"}, - {"topic": "test_topic", "key": "0", "value": "1,test"}, - {"topic": "test_topic", "key": "0", "value": "2,test"}, - {"topic": "test_topic", "key": "0", "value": "2,test"}, - {"topic": "test_topic", "key": "0", "value": "1,test"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 1, "value": "2"}, - {"topic": "OUTPUT", "key": 1, "value": "3"} - ] - }, - { - "name": "with having expression on non-group-by field (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f2, COUNT(*) FROM TEST GROUP BY f2 HAVING SUM(f1) > 10;" - ], - "inputs": [ - {"topic": "test_topic", "key": "-", "value": "5,a"}, - {"topic": "test_topic", "key": "-", "value": "10,b"}, - {"topic": "test_topic", "key": "-", "value": "6,a"}, - {"topic": "test_topic", "key": "-", "value": "1,b"}, - {"topic": "test_topic", "key": "-", "value": "-1,a"}, - {"topic": "test_topic", "key": "-", "value": "1,a"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "a", "value": "2"}, - {"topic": "OUTPUT", "key": "b", "value": "2"}, - {"topic": "OUTPUT", "key": "a", "value": null}, - {"topic": "OUTPUT", "key": "a", "value": "4"} - ] - }, - { - "name": "with constant having (stream-table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f2, SUM(f1) FROM TEST GROUP BY f2 HAVING f2='test';" - ], - "inputs": [ - {"topic": "test_topic", "key": "0", "value": "1,a"}, - {"topic": "test_topic", "key": "0", "value": "2,b"}, - {"topic": "test_topic", "key": "0", "value": "2,test"}, - {"topic": "test_topic", "key": "0", "value": "2,b"}, - {"topic": "test_topic", "key": "0", "value": "3,test"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "test", "value": "2"}, - {"topic": "OUTPUT", "key": "test", "value": "5"} - ] - }, - { - "name": "with constants in the projection (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, f1 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f1, 'some constant' as f3, COUNT(f1) FROM TEST GROUP BY f1;" - ], - "inputs": [ - {"topic": "test_topic", "key": 1, "value": "1"}, - {"topic": "test_topic", "key": 2, "value": "2"}, - {"topic": "test_topic", "key": 1, "value": "1"}, - {"topic": "test_topic", "key": 2, "value": "2"}, - {"topic": "test_topic", "key": 3, "value": "3"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 1, "value": "some constant,1"}, - {"topic": "OUTPUT", "key": 2, "value": "some constant,1"}, - {"topic": "OUTPUT", "key": 1, "value": "some constant,2"}, - {"topic": "OUTPUT", "key": 2, "value": "some constant,2"}, - {"topic": "OUTPUT", "key": 3, "value": "some constant,1"} - ] - }, - { - "name": "missing matching projection field (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT data, COUNT(*) FROM TEST GROUP BY data;" - ], - "inputs": [ - {"topic": "test_topic", "value": {"DATA": "d1"}, "timestamp": 1}, - {"topic": "test_topic", "value": {"DATA": "d2"}, "timestamp": 2}, - {"topic": "test_topic", "value": {"DATA": "d1"}, "timestamp": 3} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0": 1}, "timestamp": 1}, - {"topic": "OUTPUT", "key": "d2", "value": {"KSQL_COL_0": 1}, "timestamp": 2}, - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0": 2}, "timestamp": 3} - ] - }, - { - "name": "missing matching projection field (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT F2, COUNT(*) FROM TEST GROUP BY f2;" - ], - "inputs": [ - {"topic": "test_topic", "key": "1", "value": "1,a"}, - {"topic": "test_topic", "key": "2", "value": "2,b"}, - {"topic": "test_topic", "key": "1", "value": "1,b"}, - {"topic": "test_topic", "key": "2", "value": null}, - {"topic": "test_topic", "key": "1", "value": "1,a"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "a", "value": "1"}, - {"topic": "OUTPUT", "key": "b", "value": "1"}, - {"topic": "OUTPUT", "key": "a", "value": "0"}, - {"topic": "OUTPUT", "key": "b", "value": "2"}, - {"topic": "OUTPUT", "key": "b", "value": "1"}, - {"topic": "OUTPUT", "key": "b", "value": "0"}, - {"topic": "OUTPUT", "key": "a", "value": "1"} - ] - }, - { - "name": "duplicate fields (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(1), COUNT(*), AS_VALUE(DATA) AS COPY FROM TEST GROUP BY data;" - ], - "inputs": [ - {"topic": "test_topic", "value": {"DATA": "d1"}, "timestamp": 1}, - {"topic": "test_topic", "value": {"DATA": "d2"}, "timestamp": 2}, - {"topic": "test_topic", "value": {"DATA": "d1"}, "timestamp": 3} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0": 1, "KSQL_COL_1": 1, "COPY": "d1"}, "timestamp": 1}, - {"topic": "OUTPUT", "key": "d2", "value": {"KSQL_COL_0": 1, "KSQL_COL_1": 1, "COPY": "d2"}, "timestamp": 2}, - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0": 2, "KSQL_COL_1": 2, "COPY": "d1"}, "timestamp": 3} - ] - }, - { - "name": "duplicate udafs (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, data VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT DATA, COUNT(1), COUNT(1) FROM TEST GROUP BY data;" - ], - "inputs": [ - {"topic": "test_topic", "value": {"DATA": "d1"}, "timestamp": 1}, - {"topic": "test_topic", "value": {"DATA": "d2"}, "timestamp": 2}, - {"topic": "test_topic", "value": {"DATA": "d1"}, "timestamp": 3} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0": 1, "KSQL_COL_1": 1}, "timestamp": 1}, - {"topic": "OUTPUT", "key": "d2", "value": {"KSQL_COL_0": 1, "KSQL_COL_1": 1}, "timestamp": 2}, - {"topic": "OUTPUT", "key": "d1", "value": {"KSQL_COL_0": 2, "KSQL_COL_1": 2}, "timestamp": 3} - ] - }, - { - "name": "with non-aggregate projection field not in group by (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, d1 VARCHAR, d2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT d1, COUNT(*) FROM TEST GROUP BY d2;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: D1" - } - }, - { - "name": "with non-aggregate projection field not in group by (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, d1 VARCHAR, d2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT d1, COUNT(*) FROM TEST GROUP BY d2;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Non-aggregate SELECT expression(s) not part of GROUP BY: D1" - } - }, - { - "name": "aggregate function (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, d1 VARCHAR, d2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT COUNT(*) FROM TEST GROUP BY SUM(d2);" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "GROUP BY does not support aggregate functions: SUM is an aggregate function." - } - }, - { - "name": "aggregate function nested in arithmetic (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, d1 VARCHAR, d2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT COUNT(*) FROM TEST GROUP BY 1 + SUM(d2);" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "GROUP BY does not support aggregate functions: SUM is an aggregate function." - } - }, - { - "name": "aggregate function nested in UDF (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, d1 VARCHAR, d2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT COUNT(*) FROM TEST GROUP BY SUBSTRING(d1, SUM(d2), 1);" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "GROUP BY does not support aggregate functions: SUM is an aggregate function." - } - }, - { - "name": "without aggregate functions (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, d1 VARCHAR, d2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT SUBSTRING(d1, 1, 2) FROM TEST GROUP BY d2;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "GROUP BY requires aggregate functions in either the SELECT or HAVING clause." - } - }, - { - "name": "without group-by (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, d1 VARCHAR, d2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT d1, COUNT() FROM TEST;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Use of aggregate function COUNT requires a GROUP BY clause" - } - }, - { - "name": "UDAF nested in UDF in select expression (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, d1 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT D1, SUBSTRING('Mr Bugalicious', CAST(COUNT(*) AS INT), 1) FROM TEST GROUP BY d1;" - ], - "inputs": [ - {"topic": "test_topic", "value": "x"}, - {"topic": "test_topic", "value": "xxx"}, - {"topic": "test_topic", "value": "y"}, - {"topic": "test_topic", "value": "x"}, - {"topic": "test_topic", "value": "xxx"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "x", "value": "M"}, - {"topic": "OUTPUT", "key": "xxx", "value": "M"}, - {"topic": "OUTPUT", "key": "y", "value": "M"}, - {"topic": "OUTPUT", "key": "x", "value": "r"}, - {"topic": "OUTPUT", "key": "xxx", "value": "r"} - ] - }, - { - "name": "UDAF nested in UDF in select expression (table->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, d0 INT, d1 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT d1, SUBSTRING('Mr Bugalicious', CAST(COUNT(*) AS INT), 1) FROM TEST GROUP BY d1;" - ], - "inputs": [ - {"topic": "test_topic", "key": "0", "value": "0,x"}, - {"topic": "test_topic", "key": "1", "value": "1,x"}, - {"topic": "test_topic", "key": "2", "value": "2,xxx"}, - {"topic": "test_topic", "key": "3", "value": "3,xxx"}, - {"topic": "test_topic", "key": "1", "value": null}, - {"topic": "test_topic", "key": "2", "value": "2,yy"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "x", "value": "M"}, - {"topic": "OUTPUT", "key": "x", "value": "r"}, - {"topic": "OUTPUT", "key": "xxx", "value": "M"}, - {"topic": "OUTPUT", "key": "xxx", "value": "r"}, - {"topic": "OUTPUT", "key": "x", "value": "M"}, - {"topic": "OUTPUT", "key": "xxx", "value": "M"}, - {"topic": "OUTPUT", "key": "yy", "value": "M"} - ] - }, - { - "name": "UDF nested in UDAF in select expression (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, d1 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT d1, SUM(LEN(d1)) FROM TEST GROUP BY d1;" - ], - "inputs": [ - {"topic": "test_topic", "value": "x"}, - {"topic": "test_topic", "value": "xxx"}, - {"topic": "test_topic", "value": "y"}, - {"topic": "test_topic", "value": "x"}, - {"topic": "test_topic", "value": "xxx"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "x", "value": "1"}, - {"topic": "OUTPUT", "key": "xxx", "value": "3"}, - {"topic": "OUTPUT", "key": "y", "value": "1"}, - {"topic": "OUTPUT", "key": "x", "value": "2"}, - {"topic": "OUTPUT", "key": "xxx", "value": "6"} - ] - }, - { - "name": "UDAF nested in UDAF in select expression (stream->table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, d1 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT SUM(COUNT()) FROM TEST GROUP BY d1;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Aggregate functions can not be nested: SUM(COUNT())" - } - }, - { - "name": "should exclude any stream row whose single GROUP BY expression resolves to NULL", - "comment": "Passing NULL as the POS to SUBSTRING should resolve to NULL without an exception", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, str STRING, pos INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT SUBSTRING(str, pos), COUNT() FROM TEST GROUP BY SUBSTRING(str, pos);" - ], - "inputs": [ - {"topic": "test_topic", "value": "xx,1"}, - {"topic": "test_topic", "value": "x,"}, - {"topic": "test_topic", "value": "xx,1"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "xx", "value": "1"}, - {"topic": "OUTPUT", "key": "xx", "value": "2"} - ] - }, - { - "name": "should exclude any table row whose single GROUP BY expression resolves to NULL", - "comment": "Passing NULL as the POS to SUBSTRING should resolve to NULL without an exception", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, str STRING, pos INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT SUBSTRING(str, pos), COUNT() FROM TEST GROUP BY SUBSTRING(str, pos);" - ], - "inputs": [ - {"topic": "test_topic", "key": "1", "value": "xx,1"}, - {"topic": "test_topic", "key": "2", "value": "x,"}, - {"topic": "test_topic", "key": "3", "value": "xx,1"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "xx", "value": "1"}, - {"topic": "OUTPUT", "key": "xx", "value": "2"} - ] - }, - { - "name": "should exclude any stream row whose single GROUP BY expression throws", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (K STRING KEY, id INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT BAD_UDF(id), COUNT() FROM TEST GROUP BY BAD_UDF(id);" - ], - "inputs": [ - {"topic": "test_topic", "value": "1"} - ], - "outputs": [ - ] - }, - { - "name": "should exclude any table row whose single GROUP BY expression throws", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE TEST (K STRING PRIMARY KEY, id INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT BAD_UDF(id), COUNT() FROM TEST GROUP BY BAD_UDF(id);" - ], - "inputs": [ - {"topic": "test_topic", "key": "2", "value": "1"} - ], - "outputs": [ - ] - }, - { - "name": "by non-STRING key", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (K STRING KEY, f0 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", - "CREATE TABLE OUTPUT AS SELECT f0, COUNT(1) FROM INPUT GROUP BY f0;" - ], - "inputs": [ - {"topic": "test_topic", "value": "2"}, - {"topic": "test_topic", "value": "3"}, - {"topic": "test_topic", "value": "2"} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 2, "value": "1"}, - {"topic": "OUTPUT", "key": 3, "value": "1"}, - {"topic": "OUTPUT", "key": 2, "value": "2"} - ], - "post": { - "sources": [ - { - "name": "OUTPUT", - "type": "table", - "keyFormat": {"format": "KAFKA"}, - "schema": "F0 INTEGER KEY, KSQL_COL_0 BIGINT" - } - ] - } - }, - { - "name": "should handled quoted key and value", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (`Key` STRING KEY, IGNORED INT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT AS SELECT `Key`, COUNT(1) AS `Value` FROM INPUT GROUP BY `Key`;" - ], - "inputs": [ - {"topic": "test_topic", "timestamp": 12345, "key": "11", "value": {}}, - {"topic": "test_topic", "timestamp": 12365, "key": "10", "value": {}}, - {"topic": "test_topic", "timestamp": 12375, "key": "11", "value": {}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": "11", "value": {"Value": 1}}, - {"topic": "OUTPUT", "key": "10", "value": {"Value": 1}}, - {"topic": "OUTPUT", "key": "11", "value": {"Value": 2}} - ], - "post": { - "sources": [ - { - "name": "OUTPUT", - "type": "table", - "keyFormat": {"format": "KAFKA"}, - "schema": "`Key` STRING KEY, `Value` BIGINT" - } - ] - } - }, - { - "name": "on join", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE t1 (ID BIGINT PRIMARY KEY, TOTAL integer) WITH (kafka_topic='T1', value_format='AVRO');", - "CREATE TABLE t2 (ID BIGINT PRIMARY KEY, TOTAL integer) WITH (kafka_topic='T2', value_format='AVRO');", - "CREATE TABLE OUTPUT AS SELECT t1.ID, SUM(t1.total + CASE WHEN t2.total IS NULL THEN 0 ELSE t2.total END) as SUM FROM T1 LEFT JOIN T2 ON (t1.ID = t2.ID) GROUP BY t1.ID HAVING COUNT(1) > 0;" - ], - "inputs": [ - {"topic": "T1", "key": 0, "value": {"total": 100}}, - {"topic": "T1", "key": 1, "value": {"total": 101}}, - {"topic": "T2", "key": 0, "value": {"total": 5}}, - {"topic": "T2", "key": 1, "value": {"total": 10}}, - {"topic": "T2", "key": 0, "value": {"total": 20}}, - {"topic": "T2", "key": 0, "value": null} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 0,"value": {"SUM": 100}}, - {"topic": "OUTPUT", "key": 1,"value": {"SUM": 101}}, - {"topic": "OUTPUT", "key": 0,"value": null}, - {"topic": "OUTPUT", "key": 0,"value": {"SUM": 105}}, - {"topic": "OUTPUT", "key": 1,"value": null}, - {"topic": "OUTPUT", "key": 1,"value": {"SUM": 111}}, - {"topic": "OUTPUT", "key": 0,"value": null}, - {"topic": "OUTPUT", "key": 0,"value": {"SUM": 120}}, - {"topic": "OUTPUT", "key": 0,"value": null}, - {"topic": "OUTPUT", "key": 0,"value": {"SUM": 100}} - ] - }, - { - "name": "windowed join", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE A (id varchar primary key, regionid varchar) WITH (kafka_topic='a', value_format='json');", - "CREATE STREAM B (id varchar) WITH (kafka_topic='b', value_format='json');", - "CREATE TABLE test AS SELECT a.id, COUNT(*) as count FROM B LEFT JOIN A ON a.id = b.id WINDOW TUMBLING (SIZE 1 MINUTE) GROUP BY a.id HAVING COUNT(*) > 2;" - ], - "inputs": [ - {"topic": "a", "key": "1", "value": {"id": "1", "regionid": "one"}}, - {"topic": "b", "value": {"Id": "1"}}, - {"topic": "b", "value": {"Id": "1"}}, - {"topic": "b", "value": {"Id": "1"}} - ], - "outputs": [ - {"topic": "TEST", "key": "1", "value": {"COUNT": 3}, "window": {"start": 0, "end": 60000, "type": "time"}} - ] - }, - { - "name": "windowed join with window bounds", - "tracked by": "https://github.com/confluentinc/ksql/issues/5931", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM A (ID VARCHAR, col1 VARCHAR) WITH (kafka_topic='a', value_format='JSON');", - "CREATE TABLE B (ID VARCHAR PRIMARY KEY, col1 VARCHAR) WITH (kafka_topic='b', value_format='JSON');", - "CREATE TABLE C AS SELECT A.ID, COUNT(*), WINDOWSTART as WSTART, WINDOWEND AS WEND FROM A JOIN B on A.ID = B.ID WINDOW TUMBLING (SIZE 10 MILLISECONDS) GROUP BY a.ID;" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "SELECT column 'WINDOWSTART' cannot be resolved." - } - }, - { - "name": "zero non-agg columns (stream)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (VALUE INT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT as SELECT 1 as k, count(1) AS ID FROM INPUT group by 1;" - ], - "inputs": [ - {"topic": "test_topic", "value": {"VALUE": 0}}, - {"topic": "test_topic", "value": {"VALUE": 0}}, - {"topic": "test_topic", "value": {"VALUE": 0}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 1, "value": {"ID": 1}}, - {"topic": "OUTPUT", "key": 1, "value": {"ID": 2}}, - {"topic": "OUTPUT", "key": 1, "value": {"ID": 3}} - ] - }, - { - "name": "zero non-agg columns (windowed stream)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (VALUE INT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT as SELECT 1 as k, count(1) AS ID FROM INPUT WINDOW TUMBLING (SIZE 1 SECOND) group by 1;" - ], - "inputs": [ - {"topic": "test_topic", "value": {"VALUE": 0}}, - {"topic": "test_topic", "value": {"VALUE": 0}}, - {"topic": "test_topic", "value": {"VALUE": 0}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 1, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"ID": 1}}, - {"topic": "OUTPUT", "key": 1, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"ID": 2}}, - {"topic": "OUTPUT", "key": 1, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"ID": 3}} - ] - }, - { - "name": "zero non-agg columns (table)", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE TABLE INPUT (ID INT PRIMARY KEY, VALUE INT) WITH (kafka_topic='test_topic', value_format='JSON');", - "CREATE TABLE OUTPUT as SELECT 1 as k, count(1) AS ID FROM INPUT group by 1;" - ], - "inputs": [ - {"topic": "test_topic", "key": 10, "value": {"VALUE": 0}}, - {"topic": "test_topic", "key": 1666, "value": {"VALUE": 0}}, - {"topic": "test_topic", "key": 98, "value": {"VALUE": 0}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": 1, "value": {"ID": 1}}, - {"topic": "OUTPUT", "key": 1, "value": {"ID": 2}}, - {"topic": "OUTPUT", "key": 1, "value": {"ID": 3}} - ] - }, - { - "name": "windowed aggregate with struct key", - "format": ["JSON", "AVRO"], - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (ID STRUCT KEY, VAL INT) WITH (kafka_topic='test_topic', format='{FORMAT}');", - "CREATE TABLE OUTPUT as SELECT ID, count(1) AS count FROM INPUT WINDOW TUMBLING (SIZE 1 SECOND) group by ID;" - ], - "inputs": [ - {"topic": "test_topic", "key":{"F1": 1, "F2": 1}, "value": {"VAL": 0}}, - {"topic": "test_topic", "key":{"F1": 1, "F2": 1}, "value": {"VAL": 0}}, - {"topic": "test_topic", "key":{"F1": 1, "F2": 1}, "value": {"VAL": 0}} - ], - "outputs": [ - {"topic": "OUTPUT", "key":{"F1": 1, "F2": 1}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key":{"F1": 1, "F2": 1}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key":{"F1": 1, "F2": 1}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"COUNT": 3}} - ] - }, - { - "name": "windowed aggregate with field within struct key", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM INPUT (ID STRUCT KEY, VAL INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT as SELECT ID->F1, count(1) AS count FROM INPUT WINDOW TUMBLING (SIZE 1 SECOND) group by ID->F1;" - ], - "inputs": [ - {"topic": "test_topic", "key":{"F1": 1, "F2": 1}, "value": {"VAL": 0}}, - {"topic": "test_topic", "key":{"F1": 1, "F2": 1}, "value": {"VAL": 0}}, - {"topic": "test_topic", "key":{"F1": 1, "F2": 1}, "value": {"VAL": 0}} - ], - "outputs": [ - {"topic": "OUTPUT", "key":1, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key":1, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"COUNT": 2}}, - {"topic": "OUTPUT", "key":1, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"COUNT": 3}} - ] - }, - { - "name": "non-KAFKA key format", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, VAL BOOLEAN) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT AS SELECT VAL, COUNT() AS COUNT FROM TEST GROUP BY VAL;" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"VAL": true}}, - {"topic": "test_topic", "key": 1, "value": {"VAL": false}}, - {"topic": "test_topic", "key": 2, "value": {"VAL": true}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": true, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": false, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": true, "value": {"COUNT": 2}} - ] - }, - { - "name": "AVRO primitive key", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, VAL BOOLEAN) WITH (kafka_topic='test_topic', format='AVRO');", - "CREATE TABLE OUTPUT AS SELECT VAL, COUNT() AS COUNT FROM TEST GROUP BY VAL;" - ], - "topics": [ - { - "name": "test_topic", - "keySchema": {"type": "int"}, - "keyFormat": "AVRO", - "valueSchema": {"name": "ignored", "type": "record", "fields": [{"name": "VAL", "type": "boolean"}]}, - "valueFormat": "AVRO" - } - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"VAL": true}}, - {"topic": "test_topic", "key": 1, "value": {"VAL": false}}, - {"topic": "test_topic", "key": 2, "value": {"VAL": true}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": true, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": false, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": true, "value": {"COUNT": 2}} - ], - "post": { - "sources": [ - { - "comment": "unwrap singles is implicitly set on all key formats that support both wrapping and unwrapping, including AVRO", - "name" : "OUTPUT", - "type" : "TABLE", - "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", - "keyFormat" : { - "format" : "AVRO" - }, - "valueFormat" : "AVRO", - "keyFeatures" : [ "UNWRAP_SINGLES" ], - "valueFeatures" : [ ] - } - ] - } - }, - { - "name": "AVRO struct key group by primitive", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID STRUCT KEY, VAL BOOLEAN) WITH (kafka_topic='test_topic', format='AVRO');", - "CREATE TABLE OUTPUT AS SELECT VAL, COUNT() AS COUNT FROM TEST GROUP BY VAL;" - ], - "topics": [ - { - "name": "test_topic", - "keySchema": {"name": "key", "type": "record", "fields": [{"name": "F1", "type": ["null", "int"]}]}, - "keyFormat": "AVRO", - "valueSchema": {"name": "ignored", "type": "record", "fields": [{"name": "VAL", "type": "boolean"}]}, - "valueFormat": "AVRO" - } - ], - "inputs": [ - {"topic": "test_topic", "key": {"F1": 0}, "value": {"VAL": true}}, - {"topic": "test_topic", "key": {"F1": 0}, "value": {"VAL": false}}, - {"topic": "test_topic", "key": {"F1": 1}, "value": {"VAL": true}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": true, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": false, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": true, "value": {"COUNT": 2}} - ], - "post": { - "sources": [ - { - "comment": "unwrap singles is implicitly set on all key formats that support both wrapping and unwrapping, including AVRO", - "name" : "OUTPUT", - "type" : "TABLE", - "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", - "keyFormat" : { - "format" : "AVRO" - }, - "valueFormat" : "AVRO", - "keyFeatures" : [ "UNWRAP_SINGLES" ], - "valueFeatures" : [ ] - } - ] - } - }, - { - "name": "AVRO group by struct", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, A INT, B INT) WITH (kafka_topic='test_topic', format='AVRO');", - "CREATE TABLE OUTPUT AS SELECT STRUCT(a:=A, b:=B) AS ROWKEY, COUNT() AS COUNT FROM TEST GROUP BY STRUCT(a:=A, b:=B);" - ], - "topics": [ - { - "name": "test_topic", - "keySchema": {"type": "int"}, - "keyFormat": "AVRO", - "valueSchema": {"name": "ignored", "type": "record", "fields": [{"name": "A", "type": "int"}, {"name": "B", "type": "int"}]}, - "valueFormat": "AVRO" - } - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"A": 1, "B": 1}}, - {"topic": "test_topic", "key": 1, "value": {"A": 2, "B": 1}}, - {"topic": "test_topic", "key": 2, "value": {"A": 1, "B": 1}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"A": 1, "B": 1}, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": {"A": 2, "B": 1}, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": {"A": 1, "B": 1}, "value": {"COUNT": 2}} - ], - "post": { - "sources": [ - { - "comment": "unwrap singles is implicitly set on all key formats that support both wrapping and unwrapping, including AVRO", - "name" : "OUTPUT", - "type" : "TABLE", - "schema" : "`ROWKEY` STRUCT KEY, `COUNT` BIGINT", - "keyFormat" : { - "format" : "AVRO" - }, - "valueFormat" : "AVRO", - "keyFeatures" : [ "UNWRAP_SINGLES" ], - "valueFeatures" : [ ] - } - ], - "topics": { - "topics": [ - { - "name": "OUTPUT", - "keyFormat" : { "format" : "AVRO", "features": ["UNWRAP_SINGLES"], "properties": {"fullSchemaName": "io.confluent.ksql.avro_schemas.OutputKey"} }, - "keySchema": { - "type": "record", - "name": "OutputKey", - "namespace": "io.confluent.ksql.avro_schemas", - "fields": [ - { "name": "A", "type": [ "null", "int" ], "default": null }, - { "name": "B", "type": [ "null", "int" ], "default": null } - ] - }, - "valueFormat" : { "format" : "AVRO" }, - "valueSchema": { - "type": "record", - "name": "KsqlDataSourceSchema", - "namespace": "io.confluent.ksql.avro_schemas", - "fields": [ - { "name": "COUNT", "type": [ "null", "long" ], "default": null } - ] - } - } - ] - } - } - }, - { - "name": "JSON group by array", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, A INT, B INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT AS SELECT ARRAY[a, b] AS ROWKEY, COUNT() AS COUNT FROM TEST GROUP BY ARRAY[A, B];" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"A": 1, "B": 1}}, - {"topic": "test_topic", "key": 1, "value": {"A": 2, "B": 1}}, - {"topic": "test_topic", "key": 2, "value": {"A": 1, "B": 1}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": [1, 1], "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": [2, 1], "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": [1, 1], "value": {"COUNT": 2}} - ] - }, - { - "name": "JSON group by struct", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, A INT, B INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT AS SELECT STRUCT(a:=A, b:=B) AS ROWKEY, COUNT() AS COUNT FROM TEST GROUP BY STRUCT(a:=A, b:=B);" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"A": 1, "B": 1}}, - {"topic": "test_topic", "key": 1, "value": {"A": 2, "B": 1}}, - {"topic": "test_topic", "key": 2, "value": {"A": 1, "B": 1}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"A": 1, "B": 1}, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": {"A": 2, "B": 1}, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": {"A": 1, "B": 1}, "value": {"COUNT": 2}} - ] - }, - { - "name": "JSON group by struct convert key format", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, A INT, B INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT WITH (key_format='AVRO') AS SELECT STRUCT(a:=A, b:=B) AS ROWKEY, COUNT() AS COUNT FROM TEST GROUP BY STRUCT(a:=A, b:=B);" - ], - "inputs": [ - {"topic": "test_topic", "key": 0, "value": {"A": 1, "B": 1}}, - {"topic": "test_topic", "key": 1, "value": {"A": 2, "B": 1}}, - {"topic": "test_topic", "key": 2, "value": {"A": 1, "B": 1}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"A": 1, "B": 1}, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": {"A": 2, "B": 1}, "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": {"A": 1, "B": 1}, "value": {"COUNT": 2}} - ], - "post": { - "topics": { - "topics": [ - { - "name": "OUTPUT", - "keyFormat" : { "format" : "AVRO", "features": ["UNWRAP_SINGLES"], "properties": {"fullSchemaName": "io.confluent.ksql.avro_schemas.OutputKey"} }, - "keySchema": { - "type": "record", - "name": "OutputKey", - "namespace": "io.confluent.ksql.avro_schemas", - "fields": [ - { "name": "A", "type": [ "null", "int" ], "default": null }, - { "name": "B", "type": [ "null", "int" ], "default": null } - ] - }, - "valueFormat" : { "format" : "JSON" } - } - ] - } - } - }, - { - "name": "JSON group by struct convert to incompatible key format", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID INT KEY, A INT, B INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT WITH (key_format='DELIMITED') AS SELECT STRUCT(a:=A, b:=B) AS ROWKEY, COUNT() AS COUNT FROM TEST GROUP BY STRUCT(a:=A, b:=B);" - ], - "expectedException": { - "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The 'DELIMITED' format does not support type 'STRUCT', column: `ROWKEY`" - } - }, - { - "name": "Struct key used in aggregate expression", - "properties": { - "ksql.multicol.key.format.enabled" : true - }, - "statements": [ - "CREATE STREAM TEST (ID STRUCT KEY, VAL INT) WITH (kafka_topic='test_topic', format='JSON');", - "CREATE TABLE OUTPUT AS SELECT ID, SUM(ID->F1) AS sum FROM TEST GROUP BY ID;" - ], - "inputs": [ - {"topic": "test_topic", "key": {"F1": 1}, "value": {"VAL": 1}}, - {"topic": "test_topic", "key": {"F1": 1}, "value": {"VAL": 2}} - ], - "outputs": [ - {"topic": "OUTPUT", "key": {"F1": 1}, "value": {"SUM": 1}}, - {"topic": "OUTPUT", "key": {"F1": 1}, "value": {"SUM": 2}} - ] - } - ] -} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/group-by.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/group-by.json index 972f00a78173..123b44b4a2a9 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/group-by.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/group-by.json @@ -1,7 +1,4 @@ { - "comments": [ - "Tests covering use of the GROUP BY clause" - ], "tests": [ { "name": "only key column (stream->table)", @@ -248,7 +245,7 @@ {"topic": "test_topic", "key": 4, "value": {"data": "2-"}} ], "outputs": [ - {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 1}}, + {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 1}}, {"topic": "OUTPUT", "key": 3, "value": {"COUNT": 1}}, {"topic": "OUTPUT", "key": 2, "value": {"COUNT": 2}}, {"topic": "OUTPUT", "key": 3, "value": {"COUNT": 2}}, @@ -360,7 +357,7 @@ { "name": "function in group-by and nested function in select", "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', value_format='json');", + "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', format='json');", "CREATE TABLE OUTPUT AS SELECT INITCAP(COL1) AS G1, COL2 AS G2, TRIM(COL3) AS G3, concat(initcap(col1), col2, trim(col3)) AS foo, COUNT(*) FROM input GROUP BY INITCAP(col1), col2, TRIM(col3);" ], "inputs": [ @@ -370,16 +367,16 @@ {"topic": "test_topic", "key": 3, "value": {"col1": "smells", "col2": "like", "col3": " teen spirit "}} ], "outputs": [ - {"topic": "OUTPUT", "key": "Smells|+|like|+|teen spirit", "value": {"FOO": "Smellsliketeen spirit", "KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "The|+|man who|+|stole the world", "value": {"FOO": "Theman whostole the world", "KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "Smells|+|like|+|spring", "value": {"FOO": "Smellslikespring","KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "Smells|+|like|+|teen spirit", "value": {"FOO": "Smellsliketeen spirit","KSQL_COL_0": 2}} + {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "teen spirit"}, "value": {"FOO": "Smellsliketeen spirit", "KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"G1": "The", "G2": "man who", "G3": "stole the world"}, "value": {"FOO": "Theman whostole the world", "KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "spring"}, "value": {"FOO": "Smellslikespring","KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "teen spirit"}, "value": {"FOO": "Smellsliketeen spirit","KSQL_COL_0": 2}} ] }, { "name": "group by column in nested non-aggregate function in select", "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', value_format='json');", + "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', format='json');", "CREATE TABLE OUTPUT AS SELECT INITCAP(COL1) AS G1, COL2 AS G2, COL3 AS G3, concat(initcap(col1), col2, trim(col3)) AS foo, COUNT(*) FROM input GROUP BY INITCAP(col1), col2, col3;" ], "inputs": [ @@ -389,16 +386,16 @@ {"topic": "test_topic", "key": 3, "value": {"col1": "smells", "col2": "like", "col3": " teen spirit "}} ], "outputs": [ - {"topic": "OUTPUT", "key": "Smells|+|like|+|teen spirit", "value": {"FOO": "Smellsliketeen spirit", "KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "The|+|man who|+|stole the world", "value": {"FOO": "Theman whostole the world", "KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "Smells|+|like|+|spring", "value": {"FOO": "Smellslikespring","KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "Smells|+|like|+| teen spirit ", "value": {"FOO": "Smellsliketeen spirit","KSQL_COL_0": 1}} + {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "teen spirit"}, "value": {"FOO": "Smellsliketeen spirit", "KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"G1": "The", "G2": "man who", "G3": "stole the world"}, "value": {"FOO": "Theman whostole the world", "KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "spring"}, "value": {"FOO": "Smellslikespring","KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": " teen spirit "}, "value": {"FOO": "Smellsliketeen spirit","KSQL_COL_0": 1}} ] }, { "name": "function group by column used in non-aggregate function in having", "statements": [ - "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', value_format='json');", + "CREATE STREAM INPUT (id INT KEY, col1 VARCHAR, col2 VARCHAR, col3 VARCHAR) WITH (kafka_topic='test_topic', format='json');", "CREATE TABLE OUTPUT AS SELECT INITCAP(COL1) AS G1, COL2 AS G2, trim(COL3) AS G3, COUNT(*) FROM input GROUP BY INITCAP(col1), col2, trim(col3) HAVING substring(trim(col3),1,4) = 'teen';" ], "inputs": [ @@ -408,8 +405,8 @@ {"topic": "test_topic", "key": 3, "value": {"col1": "smells", "col2": "like", "col3": " teen spirit "}} ], "outputs": [ - {"topic": "OUTPUT", "key": "Smells|+|like|+|teen spirit", "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "Smells|+|like|+|teen spirit", "value": {"KSQL_COL_0": 2}} + {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "teen spirit"}, "value": {"KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"G1": "Smells", "G2": "like", "G3": "teen spirit"}, "value": {"KSQL_COL_0": 2}} ] }, { @@ -483,7 +480,7 @@ { "name": "multiple expressions", "statements": [ - "CREATE STREAM TEST (f1 INT KEY, f2 INT) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (f1 INT KEY, f2 INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT f2, f1, f2+f1, COUNT(*) FROM TEST GROUP BY f1, f2;" ], "inputs": [ @@ -494,18 +491,55 @@ {"topic": "test_topic", "key": 2, "value": {"f2": "1"}} ], "outputs": [ - {"topic": "OUTPUT", "key": "1|+|2", "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": "2|+|4", "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": "1|+|2", "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 2}}, - {"topic": "OUTPUT", "key": "2|+|4", "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 2}}, - {"topic": "OUTPUT", "key": "2|+|1", "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}} + {"topic": "OUTPUT", "key": {"F1": 1, "F2": 2}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}}, + {"topic": "OUTPUT", "key": {"F1": 2, "F2": 4}, "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 1}}, + {"topic": "OUTPUT", "key": {"F1": 1, "F2": 2}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 2}}, + {"topic": "OUTPUT", "key": {"F1": 2, "F2": 4}, "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 2}}, + {"topic": "OUTPUT", "key": {"F1": 2, "F2": 1}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_2 STRING KEY, KSQL_COL_0 INT, KSQL_COL_1 BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "F1 INT KEY, F2 INT KEY, KSQL_COL_0 INT, KSQL_COL_1 BIGINT"} ] } }, + { + "name": "multiple expressions with struct field and other expression", + "statements": [ + "CREATE STREAM TEST (f1 INT KEY, f2 INT, address STRUCT) WITH (kafka_topic='test_topic', format='JSON');", + "CREATE TABLE OUTPUT AS SELECT address->town, f1, 2*f2+f1, COUNT(*), 2*f2 FROM TEST GROUP BY f1, address->town, 2*f2;" + ], + "inputs": [ + {"topic": "test_topic", "key": 1, "value": {"f2": "2", "ADDRESS": {"STREET": "1st Street", "Town": "Oxford"}}}, + {"topic": "test_topic", "key": 2, "value": {"f2": "4", "ADDRESS": {"STREET": "1st Street", "Town": "London"}}}, + {"topic": "test_topic", "key": 1, "value": {"f2": "2", "ADDRESS": {"STREET": "1st Street", "Town": "Oxford"}}}, + {"topic": "test_topic", "key": 2, "value": {"f2": "4", "ADDRESS": {"STREET": "1st Street", "Town": "London"}}}, + {"topic": "test_topic", "key": 2, "value": {"f2": "1", "ADDRESS": {"STREET": "1st Street", "Town": "Oxford"}}} + ], + "outputs": [ + {"topic": "OUTPUT", "key": {"F1": 1, "TOWN": "Oxford", "KSQL_COL_2": 4}, "value": {"KSQL_COL_0": 5, "KSQL_COL_1": 1}}, + {"topic": "OUTPUT", "key": {"F1": 2, "TOWN": "London", "KSQL_COL_2": 8}, "value": {"KSQL_COL_0": 10, "KSQL_COL_1": 1}}, + {"topic": "OUTPUT", "key": {"F1": 1, "TOWN": "Oxford", "KSQL_COL_2": 4}, "value": {"KSQL_COL_0": 5, "KSQL_COL_1": 2}}, + {"topic": "OUTPUT", "key": {"F1": 2, "TOWN": "London", "KSQL_COL_2": 8}, "value": {"KSQL_COL_0": 10, "KSQL_COL_1": 2}}, + {"topic": "OUTPUT", "key": {"F1": 2, "TOWN": "Oxford", "KSQL_COL_2": 2}, "value": {"KSQL_COL_0": 4, "KSQL_COL_1": 1}} + ], + "post": { + "sources": [ + {"name": "OUTPUT", "type": "table", "schema": "F1 INT KEY, TOWN STRING KEY, KSQL_COL_2 INT KEY, KSQL_COL_0 INT, KSQL_COL_1 BIGINT"} + ] + } + }, + { + "name": "multiple expressions - KAFKA key format", + "statements": [ + "CREATE STREAM TEST (f1 INT KEY, f2 INT) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE TABLE OUTPUT AS SELECT f2, f1, f2+f1, COUNT(*) FROM TEST GROUP BY f1, f2;" + ], + "expectedException": { + "type": "io.confluent.ksql.util.KsqlStatementException", + "message": "Key format does not support schema.\nformat: KAFKA\nschema: Persistence{columns=[`F1` INTEGER KEY, `F2` INTEGER KEY], features=[]}\nreason: The 'KAFKA' format only supports a single field. Got: [`F1` INTEGER KEY, `F2` INTEGER KEY]\nStatement: CREATE TABLE OUTPUT AS SELECT\n TEST.F2 F2,\n TEST.F1 F1,\n (TEST.F2 + TEST.F1) KSQL_COL_0,\n COUNT(*) KSQL_COL_1\nFROM TEST TEST\nGROUP BY TEST.F1, TEST.F2\nEMIT CHANGES" + } + }, { "name": "multiple expressions - windowed", "statements": [ @@ -520,15 +554,15 @@ {"topic": "test_topic", "key": 2, "value": {"f2": "1"}} ], "outputs": [ - {"topic": "OUTPUT", "key": "1|+|2", "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": "2|+|4", "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 1}}, - {"topic": "OUTPUT", "key": "1|+|2", "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 2}}, - {"topic": "OUTPUT", "key": "2|+|4", "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 2}}, - {"topic": "OUTPUT", "key": "2|+|1", "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}} + {"topic": "OUTPUT", "key": {"F1": 1, "F2": 2}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}}, + {"topic": "OUTPUT", "key": {"F1": 2, "F2": 4}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 1}}, + {"topic": "OUTPUT", "key": {"F1": 1, "F2": 2}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 2}}, + {"topic": "OUTPUT", "key": {"F1": 2, "F2": 4}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 6, "KSQL_COL_1": 2}}, + {"topic": "OUTPUT", "key": {"F1": 2, "F2": 1}, "window": {"start": 0, "end": 1000, "type": "time"}, "value": {"KSQL_COL_0": 3, "KSQL_COL_1": 1}} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_2 STRING KEY, KSQL_COL_0 INT, KSQL_COL_1 BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "F1 INT KEY, F2 INT KEY, KSQL_COL_0 INT, KSQL_COL_1 BIGINT"} ] } }, @@ -544,13 +578,13 @@ {"topic": "test_topic", "key": 98, "value": {"VALUE": 0}} ], "outputs": [ - {"topic": "OUTPUT", "key": "0|+|1", "value": {"ID": 1}}, - {"topic": "OUTPUT", "key": "0|+|1", "value": {"ID": 2}}, - {"topic": "OUTPUT", "key": "0|+|1", "value": {"ID": 3}} + {"topic": "OUTPUT", "key": {"VALUE": 0, "K": 1}, "value": {"ID": 1}}, + {"topic": "OUTPUT", "key": {"VALUE": 0, "K": 1}, "value": {"ID": 2}}, + {"topic": "OUTPUT", "key": {"VALUE": 0, "K": 1}, "value": {"ID": 3}} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_0 STRING KEY, ID BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "VALUE INT KEY, K INT KEY, ID BIGINT"} ] } }, @@ -623,14 +657,14 @@ { "name": "select * where all columns in group by", "statements": [ - "CREATE STREAM TEST (id INT KEY, id2 INT) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (id INT KEY, id2 INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT *, COUNT() FROM TEST GROUP BY id, id2;" ], "inputs": [ {"topic": "test_topic", "key": 1, "value": {"ID2": 2}} ], "outputs": [ - {"topic": "OUTPUT", "key": "1|+|2", "value": {"KSQL_COL_0": 1}} + {"topic": "OUTPUT", "key": {"ID": 1, "ID2": 2}, "value": {"KSQL_COL_0": 1}} ] }, { @@ -669,7 +703,7 @@ { "name": "complex UDAF params", "statements": [ - "CREATE STREAM TEST (V0 INT KEY, V1 INT) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (V0 INT KEY, V1 INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT V0, V1, SUM(V0 + V1) AS SUM FROM TEST GROUP BY V0, V1;" ], "inputs": [ @@ -678,13 +712,13 @@ {"topic": "test_topic", "key": 0, "value": {"V1": 10}} ], "outputs": [ - {"topic": "OUTPUT", "key": "0|+|10", "value": {"SUM": 10}}, - {"topic": "OUTPUT", "key": "1|+|20", "value": {"SUM": 21}}, - {"topic": "OUTPUT", "key": "0|+|10", "value": {"SUM": 20}} + {"topic": "OUTPUT", "key": {"V0": 0, "V1": 10}, "value": {"SUM": 10}}, + {"topic": "OUTPUT", "key": {"V0": 1, "V1": 20}, "value": {"SUM": 21}}, + {"topic": "OUTPUT", "key": {"V0": 0, "V1": 10}, "value": {"SUM": 20}} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_0 STRING KEY, SUM INT"} + {"name": "OUTPUT", "type": "table", "schema": "V0 INTEGER KEY, V1 INTEGER KEY, SUM INT"} ] } }, @@ -713,7 +747,7 @@ { "name": "complex UDAF params matching HAVING", "statements": [ - "CREATE STREAM TEST (V0 INT KEY, V1 INT) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (V0 INT KEY, V1 INT) WITH (kafka_topic='test_topic', format='JSON');", "CREATE TABLE OUTPUT AS SELECT SUM(V0 + V1) AS SUM, V0, V1 FROM TEST GROUP BY V0, V1 HAVING V0 + V1 <= 20;" ], "inputs": [ @@ -722,12 +756,12 @@ {"topic": "test_topic", "key": 0, "value": {"V1": 10}} ], "outputs": [ - {"topic": "OUTPUT", "key": "0|+|10", "value": {"SUM": 10}}, - {"topic": "OUTPUT", "key": "0|+|10", "value": {"SUM": 20}} + {"topic": "OUTPUT", "key": {"V0": 0, "V1": 10}, "value": {"SUM": 10}}, + {"topic": "OUTPUT", "key": {"V0": 0, "V1": 10}, "value": {"SUM": 20}} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_0 STRING KEY, SUM INT"} + {"name": "OUTPUT", "type": "table", "schema": "V0 INTEGER KEY, V1 INTEGER KEY, SUM INT"} ] } }, @@ -883,33 +917,33 @@ { "name": "fields (stream->table)", "statements": [ - "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", + "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], "inputs": [ - {"topic": "test_topic", "key": 1, "value": "1,a"}, - {"topic": "test_topic", "key": 2, "value": "2,b"}, - {"topic": "test_topic", "key": 1, "value": "1,a"}, - {"topic": "test_topic", "key": 2, "value": "2,b"}, - {"topic": "test_topic", "key": 3, "value": "3,a"} + {"topic": "test_topic", "key": "1", "value": "1,a"}, + {"topic": "test_topic", "key": "2", "value": "2,b"}, + {"topic": "test_topic", "key": "1", "value": "1,a"}, + {"topic": "test_topic", "key": "2", "value": "2,b"}, + {"topic": "test_topic", "key": "3", "value": "3,a"} ], "outputs": [ - {"topic": "OUTPUT", "key": "a|+|1", "value": "1"}, - {"topic": "OUTPUT", "key": "b|+|2", "value": "1"}, - {"topic": "OUTPUT", "key": "a|+|1", "value": "2"}, - {"topic": "OUTPUT", "key": "b|+|2", "value": "2"}, - {"topic": "OUTPUT", "key": "a|+|3", "value": "1"} + {"topic": "OUTPUT", "key": "a,1", "value": "1"}, + {"topic": "OUTPUT", "key": "b,2", "value": "1"}, + {"topic": "OUTPUT", "key": "a,1", "value": "2"}, + {"topic": "OUTPUT", "key": "b,2", "value": "2"}, + {"topic": "OUTPUT", "key": "a,3", "value": "1"} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_1 STRING KEY, KSQL_COL_0 BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "F2 STRING KEY, F1 INTEGER KEY, KSQL_COL_0 BIGINT"} ] } }, { "name": "fields used in expression", "statements": [ - "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", + "CREATE STREAM TEST (K STRING KEY, f1 INT, f2 INT) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT F1, F2, f1 / f2, COUNT(*) FROM TEST GROUP BY f1, f2;" ], "inputs": [ @@ -918,20 +952,20 @@ {"topic": "test_topic", "value": "9,3"} ], "outputs": [ - {"topic": "OUTPUT", "key": "4|+|2", "value": "2,1"}, - {"topic": "OUTPUT", "key": "9|+|3", "value": "3,1"}, - {"topic": "OUTPUT", "key": "9|+|3", "value": "3,2"} + {"topic": "OUTPUT", "key": "4,2", "value": "2,1"}, + {"topic": "OUTPUT", "key": "9,3", "value": "3,1"}, + {"topic": "OUTPUT", "key": "9,3", "value": "3,2"} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_2 STRING KEY, KSQL_COL_0 INTEGER, KSQL_COL_1 BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "F1 INTEGER KEY, F2 INTEGER KEY, KSQL_COL_0 INTEGER, KSQL_COL_1 BIGINT"} ] } }, { "name": "fields (stream->table) - format", "statements": [ - "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", + "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', key_format='JSON', value_format='{FORMAT}');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], "format": ["AVRO", "JSON", "PROTOBUF"], @@ -943,15 +977,15 @@ {"topic": "test_topic", "key": 3, "value": {"F1": 3, "F2": "a"}} ], "outputs": [ - {"topic": "OUTPUT", "key": "a|+|1", "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "b|+|2", "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "a|+|1", "value": {"KSQL_COL_0": 2}}, - {"topic": "OUTPUT", "key": "b|+|2", "value": {"KSQL_COL_0": 2}}, - {"topic": "OUTPUT", "key": "a|+|3", "value": {"KSQL_COL_0": 1}} + {"topic": "OUTPUT", "key": {"F2": "a", "F1": 1}, "value": {"KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"F2": "b", "F1": 2}, "value": {"KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"F2": "a", "F1": 1}, "value": {"KSQL_COL_0": 2}}, + {"topic": "OUTPUT", "key": {"F2": "b", "F1": 2}, "value": {"KSQL_COL_0": 2}}, + {"topic": "OUTPUT", "key": {"F2": "a", "F1": 3}, "value": {"KSQL_COL_0": 1}} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_1 STRING KEY, KSQL_COL_0 BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "F2 STRING KEY, F1 INTEGER KEY, KSQL_COL_0 BIGINT"} ] } }, @@ -980,89 +1014,89 @@ { "name": "with single grouping set (stream->table)", "statements": [ - "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR, f3 INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", + "CREATE STREAM TEST (ID INT KEY, f1 INT, f2 VARCHAR, f3 INT) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, f2, f3, COUNT(*) FROM TEST GROUP BY (f3, f2, f1);" ], "inputs": [ - {"topic": "test_topic", "key": 1, "value": "1,a,-1"}, - {"topic": "test_topic", "key": 2, "value": "2,b,-2"}, - {"topic": "test_topic", "key": 1, "value": "1,a,-1"}, - {"topic": "test_topic", "key": 2, "value": "2,b,-2"}, - {"topic": "test_topic", "key": 3, "value": "3,a,-3"} + {"topic": "test_topic", "key": "1", "value": "1,a,-1"}, + {"topic": "test_topic", "key": "2", "value": "2,b,-2"}, + {"topic": "test_topic", "key": "1", "value": "1,a,-1"}, + {"topic": "test_topic", "key": "2", "value": "2,b,-2"}, + {"topic": "test_topic", "key": "3", "value": "3,a,-3"} ], "outputs": [ - {"topic": "OUTPUT", "key": "-1|+|a|+|1", "value": "1"}, - {"topic": "OUTPUT", "key": "-2|+|b|+|2", "value": "1"}, - {"topic": "OUTPUT", "key": "-1|+|a|+|1", "value": "2"}, - {"topic": "OUTPUT", "key": "-2|+|b|+|2", "value": "2"}, - {"topic": "OUTPUT", "key": "-3|+|a|+|3", "value": "1"} + {"topic": "OUTPUT", "key": "\"-1\",a,1", "value": "1"}, + {"topic": "OUTPUT", "key": "\"-2\",b,2", "value": "1"}, + {"topic": "OUTPUT", "key": "\"-1\",a,1", "value": "2"}, + {"topic": "OUTPUT", "key": "\"-2\",b,2", "value": "2"}, + {"topic": "OUTPUT", "key": "\"-3\",a,3", "value": "1"} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_1 STRING KEY, KSQL_COL_0 BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "F3 INTEGER KEY, F2 STRING KEY, F1 INTEGER KEY, KSQL_COL_0 BIGINT"} ] } }, { "name": "fields (table->table)", "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", + "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], "inputs": [ - {"topic": "test_topic", "key": 1, "value": "1,a"}, - {"topic": "test_topic", "key": 2, "value": "2,b"}, - {"topic": "test_topic", "key": 1, "value": "1,b"}, - {"topic": "test_topic", "key": 2, "value": null}, - {"topic": "test_topic", "key": 1, "value": "1,a"} + {"topic": "test_topic", "key": "1", "value": "1,a"}, + {"topic": "test_topic", "key": "2", "value": "2,b"}, + {"topic": "test_topic", "key": "1", "value": "1,b"}, + {"topic": "test_topic", "key": "2", "value": null}, + {"topic": "test_topic", "key": "1", "value": "1,a"} ], "outputs": [ - {"topic": "OUTPUT", "key": "a|+|1", "value": "1"}, - {"topic": "OUTPUT", "key": "b|+|2", "value": "1"}, - {"topic": "OUTPUT", "key": "a|+|1", "value": "0"}, - {"topic": "OUTPUT", "key": "b|+|1", "value": "1"}, - {"topic": "OUTPUT", "key": "b|+|2", "value": "0"}, - {"topic": "OUTPUT", "key": "b|+|1", "value": "0"}, - {"topic": "OUTPUT", "key": "a|+|1", "value": "1"} + {"topic": "OUTPUT", "key": "a,1", "value": "1"}, + {"topic": "OUTPUT", "key": "b,2", "value": "1"}, + {"topic": "OUTPUT", "key": "a,1", "value": "0"}, + {"topic": "OUTPUT", "key": "b,1", "value": "1"}, + {"topic": "OUTPUT", "key": "b,2", "value": "0"}, + {"topic": "OUTPUT", "key": "b,1", "value": "0"}, + {"topic": "OUTPUT", "key": "a,1", "value": "1"} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_1 STRING KEY, KSQL_COL_0 BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "F2 STRING KEY, F1 INTEGER KEY, KSQL_COL_0 BIGINT"} ] } }, { "name": "fields - copied into value (table->table)", "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='DELIMITED');", + "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT f1, f2, AS_VALUE(f1) AS F3, AS_VALUE(F2) AS F4, COUNT(*) FROM TEST GROUP BY f2, f1;" ], "inputs": [ - {"topic": "test_topic", "key": 1, "value": "1,a"}, - {"topic": "test_topic", "key": 2, "value": "2,b"}, - {"topic": "test_topic", "key": 1, "value": "1,b"}, - {"topic": "test_topic", "key": 2, "value": null}, - {"topic": "test_topic", "key": 1, "value": "1,a"} + {"topic": "test_topic", "key": "1", "value": "1,a"}, + {"topic": "test_topic", "key": "2", "value": "2,b"}, + {"topic": "test_topic", "key": "1", "value": "1,b"}, + {"topic": "test_topic", "key": "2", "value": null}, + {"topic": "test_topic", "key": "1", "value": "1,a"} ], "outputs": [ - {"topic": "OUTPUT", "key": "a|+|1", "value": "1,a,1"}, - {"topic": "OUTPUT", "key": "b|+|2", "value": "2,b,1"}, - {"topic": "OUTPUT", "key": "a|+|1", "value": "1,a,0"}, - {"topic": "OUTPUT", "key": "b|+|1", "value": "1,b,1"}, - {"topic": "OUTPUT", "key": "b|+|2", "value": "2,b,0"}, - {"topic": "OUTPUT", "key": "b|+|1", "value": "1,b,0"}, - {"topic": "OUTPUT", "key": "a|+|1", "value": "1,a,1"} + {"topic": "OUTPUT", "key": "a,1", "value": "1,a,1"}, + {"topic": "OUTPUT", "key": "b,2", "value": "2,b,1"}, + {"topic": "OUTPUT", "key": "a,1", "value": "1,a,0"}, + {"topic": "OUTPUT", "key": "b,1", "value": "1,b,1"}, + {"topic": "OUTPUT", "key": "b,2", "value": "2,b,0"}, + {"topic": "OUTPUT", "key": "b,1", "value": "1,b,0"}, + {"topic": "OUTPUT", "key": "a,1", "value": "1,a,1"} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_1 STRING KEY, F3 INT, F4 STRING, KSQL_COL_0 BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "F2 STRING KEY, F1 INTEGER KEY, F3 INT, F4 STRING, KSQL_COL_0 BIGINT"} ] } }, { "name": "fields (table->table) - format", "statements": [ - "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", + "CREATE TABLE TEST (ID INT PRIMARY KEY, f1 INT, f2 VARCHAR) WITH (kafka_topic='test_topic', key_format='JSON', value_format='{FORMAT}');", "CREATE TABLE OUTPUT AS SELECT f1, f2, COUNT(*) FROM TEST GROUP BY f2, f1;" ], "format": ["AVRO", "JSON", "PROTOBUF"], @@ -1074,17 +1108,17 @@ {"topic": "test_topic", "key": 1, "value": {"F1": 1, "F2": "a"}} ], "outputs": [ - {"topic": "OUTPUT", "key": "a|+|1", "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "b|+|2", "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "a|+|1", "value": {"KSQL_COL_0": 0}}, - {"topic": "OUTPUT", "key": "b|+|1", "value": {"KSQL_COL_0": 1}}, - {"topic": "OUTPUT", "key": "b|+|2", "value": {"KSQL_COL_0": 0}}, - {"topic": "OUTPUT", "key": "b|+|1", "value": {"KSQL_COL_0": 0}}, - {"topic": "OUTPUT", "key": "a|+|1", "value": {"KSQL_COL_0": 1}} + {"topic": "OUTPUT", "key": {"F2": "a", "F1": 1}, "value": {"KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"F2": "b", "F1": 2}, "value": {"KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"F2": "a", "F1": 1}, "value": {"KSQL_COL_0": 0}}, + {"topic": "OUTPUT", "key": {"F2": "b", "F1": 1}, "value": {"KSQL_COL_0": 1}}, + {"topic": "OUTPUT", "key": {"F2": "b", "F1": 2}, "value": {"KSQL_COL_0": 0}}, + {"topic": "OUTPUT", "key": {"F2": "b", "F1": 1}, "value": {"KSQL_COL_0": 0}}, + {"topic": "OUTPUT", "key": {"F2": "a", "F1": 1}, "value": {"KSQL_COL_0": 1}} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_1 STRING KEY, KSQL_COL_0 BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "F2 STRING KEY, F1 INTEGER KEY, KSQL_COL_0 BIGINT"} ] } }, @@ -1246,7 +1280,7 @@ { "name": "with aggregate arithmetic involving source field (stream->table)", "statements": [ - "CREATE STREAM TEST (K STRING KEY, ITEM INT, COST INT) WITH (kafka_topic='test_topic', value_format='DELIMITED');", + "CREATE STREAM TEST (K STRING KEY, ITEM INT, COST INT) WITH (kafka_topic='test_topic', format='DELIMITED');", "CREATE TABLE OUTPUT AS SELECT ITEM, COST, COST * COUNT() FROM TEST GROUP BY ITEM, COST;" ], "inputs": [ @@ -1256,10 +1290,10 @@ {"topic": "test_topic", "value": "1,10"} ], "outputs": [ - {"topic": "OUTPUT", "key": "1|+|10", "value": "10"}, - {"topic": "OUTPUT", "key": "1|+|20", "value": "20"}, - {"topic": "OUTPUT", "key": "2|+|30", "value": "30"}, - {"topic": "OUTPUT", "key": "1|+|10", "value": "20"} + {"topic": "OUTPUT", "key": "1,10", "value": "10"}, + {"topic": "OUTPUT", "key": "1,20", "value": "20"}, + {"topic": "OUTPUT", "key": "2,30", "value": "30"}, + {"topic": "OUTPUT", "key": "1,10", "value": "20"} ] }, { diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/multi-col-keys.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/multi-col-keys.json index 2822995dfdcf..fad5affcdb58 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/multi-col-keys.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/multi-col-keys.json @@ -142,11 +142,11 @@ {"topic": "input_topic", "key": {"K": 1, "K2": 2}, "value": {"V": 0}} ], "outputs": [ - {"topic": "OUTPUT", "key": "1|+|2", "value": {"COUNT": 1}} + {"topic": "OUTPUT", "key": {"K": 1, "K2": 2}, "value": {"COUNT": 1}} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_0 STRING KEY, COUNT BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "K INTEGER KEY, K2 INTEGER KEY, COUNT BIGINT"} ] } }, @@ -176,8 +176,8 @@ {"topic": "input_topic", "key": {"K": 1, "K2": 2}, "value": null} ], "outputs": [ - {"topic": "OUTPUT", "key": "1|+|2", "value": {"COUNT": 1}}, - {"topic": "OUTPUT", "key": "1|+|2", "value": {"COUNT": 0}} + {"topic": "OUTPUT", "key": {"K": 1, "K2": 2}, "value": {"COUNT": 1}}, + {"topic": "OUTPUT", "key": {"K": 1, "K2": 2}, "value": {"COUNT": 0}} ] }, { @@ -190,11 +190,11 @@ {"topic": "input_topic", "key": {"K": 1, "K2": 2}, "value": {"V": 0}} ], "outputs": [ - {"topic": "OUTPUT", "key": "1|+|2", "value": {"KV": 1, "KV2": 2, "COUNT": 1}} + {"topic": "OUTPUT", "key": {"K": 1, "K2": 2}, "value": {"KV": 1, "KV2": 2, "COUNT": 1}} ], "post": { "sources": [ - {"name": "OUTPUT", "type": "table", "schema": "KSQL_COL_0 STRING KEY, KV INT, KV2 INT, COUNT BIGINT"} + {"name": "OUTPUT", "type": "table", "schema": "K INTEGER KEY, K2 INTEGER KEY, KV INT, KV2 INT, COUNT BIGINT"} ] } }, diff --git a/ksqldb-functional-tests/src/test/resources/sql-tests/query-upgrades/prohibited-upgrades.sql b/ksqldb-functional-tests/src/test/resources/sql-tests/query-upgrades/prohibited-upgrades.sql index f8bc99d34a51..67f4755d3cde 100644 --- a/ksqldb-functional-tests/src/test/resources/sql-tests/query-upgrades/prohibited-upgrades.sql +++ b/ksqldb-functional-tests/src/test/resources/sql-tests/query-upgrades/prohibited-upgrades.sql @@ -37,21 +37,8 @@ CREATE OR REPLACE TABLE b AS SELECT col2, COUNT(*) FROM a GROUP BY col2; ---------------------------------------------------------------------------------------------------- --@test: GROUP BY - change grouping column ordering --@expected.error: io.confluent.ksql.util.KsqlException ---@expected.message: StreamGroupByV1 must have matching group by clause. Values differ: [COL1, COL2] vs. [COL2, COL1] ----------------------------------------------------------------------------------------------------- -SET 'ksql.create.or.replace.enabled' = 'true'; - -CREATE STREAM a (id INT KEY, col1 INT, col2 INT) WITH (kafka_topic='a', value_format='JSON'); -CREATE TABLE b AS SELECT col1, col2, COUNT(*) FROM a GROUP BY col1, col2; - -CREATE OR REPLACE TABLE b AS SELECT col1, col2, COUNT(*) FROM a GROUP BY col2, col1; - ----------------------------------------------------------------------------------------------------- ---@test: GROUP BY - change grouping column ordering - multicol enabled ---@expected.error: io.confluent.ksql.util.KsqlException --@expected.message: (Key columns must be identical. The following key columns are changed, missing or reordered: [`COL1` INTEGER KEY, `COL2` INTEGER KEY]) ---------------------------------------------------------------------------------------------------- -SET 'ksql.multicol.key.format.enabled' = 'true'; SET 'ksql.create.or.replace.enabled' = 'true'; CREATE STREAM a (id INT KEY, col1 INT, col2 INT) WITH (kafka_topic='a', format='JSON'); diff --git a/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ExecutionStepFactory.java b/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ExecutionStepFactory.java index 1969063a73f2..02f26c2f98a1 100644 --- a/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ExecutionStepFactory.java +++ b/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ExecutionStepFactory.java @@ -33,7 +33,6 @@ import io.confluent.ksql.execution.plan.StreamFlatMap; import io.confluent.ksql.execution.plan.StreamGroupBy; import io.confluent.ksql.execution.plan.StreamGroupByKey; -import io.confluent.ksql.execution.plan.StreamGroupByV1; import io.confluent.ksql.execution.plan.StreamSelect; import io.confluent.ksql.execution.plan.StreamSelectKey; import io.confluent.ksql.execution.plan.StreamSink; @@ -44,7 +43,6 @@ import io.confluent.ksql.execution.plan.TableAggregate; import io.confluent.ksql.execution.plan.TableFilter; import io.confluent.ksql.execution.plan.TableGroupBy; -import io.confluent.ksql.execution.plan.TableGroupByV1; import io.confluent.ksql.execution.plan.TableSelect; import io.confluent.ksql.execution.plan.TableSelectKey; import io.confluent.ksql.execution.plan.TableSink; @@ -356,21 +354,6 @@ public static StreamWindowedAggregate streamWindowedAggregate( ); } - public static StreamGroupByV1 streamGroupByV1( - final Stacker stacker, - final ExecutionStep> sourceStep, - final Formats format, - final List groupingExpressions - ) { - final QueryContext queryContext = stacker.getQueryContext(); - return new StreamGroupByV1<>( - new ExecutionStepPropertiesV1(queryContext), - sourceStep, - format, - groupingExpressions - ); - } - public static StreamGroupBy streamGroupBy( final Stacker stacker, final ExecutionStep> sourceStep, @@ -412,21 +395,6 @@ public static TableAggregate tableAggregate( ); } - public static TableGroupByV1 tableGroupByV1( - final QueryContext.Stacker stacker, - final ExecutionStep> sourceStep, - final Formats format, - final List groupingExpressions - ) { - final QueryContext queryContext = stacker.getQueryContext(); - return new TableGroupByV1<>( - new ExecutionStepPropertiesV1(queryContext), - sourceStep, - format, - groupingExpressions - ); - } - public static TableGroupBy tableGroupBy( final QueryContext.Stacker stacker, final ExecutionStep> sourceStep,