diff --git a/ksqldb-api-client/src/test/java/io/confluent/ksql/api/client/integration/ClientIntegrationTest.java b/ksqldb-api-client/src/test/java/io/confluent/ksql/api/client/integration/ClientIntegrationTest.java index 48add6a88b4c..d7c766ff5111 100644 --- a/ksqldb-api-client/src/test/java/io/confluent/ksql/api/client/integration/ClientIntegrationTest.java +++ b/ksqldb-api-client/src/test/java/io/confluent/ksql/api/client/integration/ClientIntegrationTest.java @@ -19,7 +19,6 @@ import static io.confluent.ksql.api.client.util.ClientTestUtil.subscribeAndWait; import static io.confluent.ksql.test.util.AssertEventually.assertThatEventually; import static io.confluent.ksql.util.KsqlConfig.KSQL_DEFAULT_KEY_FORMAT_CONFIG; -import static io.confluent.ksql.util.KsqlConfig.KSQL_KEY_FORMAT_ENABLED; import static io.confluent.ksql.util.KsqlConfig.KSQL_STREAMS_PREFIX; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; @@ -179,7 +178,6 @@ public class ClientIntegrationTest { private static final TestKsqlRestApp REST_APP = TestKsqlRestApp .builder(TEST_HARNESS::kafkaBootstrapServers) .withProperty(KSQL_STREAMS_PREFIX + StreamsConfig.NUM_STREAM_THREADS_CONFIG, 1) - .withProperty(KSQL_KEY_FORMAT_ENABLED, true) .withProperty(KSQL_DEFAULT_KEY_FORMAT_CONFIG, "JSON") .build(); diff --git a/ksqldb-engine/src/test/java/io/confluent/ksql/engine/KsqlEngineTest.java b/ksqldb-engine/src/test/java/io/confluent/ksql/engine/KsqlEngineTest.java index ba1a00e31a29..aba0368ceee9 100644 --- a/ksqldb-engine/src/test/java/io/confluent/ksql/engine/KsqlEngineTest.java +++ b/ksqldb-engine/src/test/java/io/confluent/ksql/engine/KsqlEngineTest.java @@ -1470,7 +1470,7 @@ public void shouldThrowOnUnsupportedKeyFormatForCreateSource() { // Given: givenTopicsExist("foo"); final PreparedStatement prepared = - prepare(parse("CREATE STREAM FOO (a int) WITH (kafka_topic='foo', value_format='json', key_format='avro');").get(0)); + prepare(parse("CREATE STREAM FOO (a int) WITH (kafka_topic='foo', value_format='json', key_format='protobuf');").get(0)); // When: final Exception e = assertThrows( @@ -1482,7 +1482,7 @@ public void shouldThrowOnUnsupportedKeyFormatForCreateSource() { ); // Then: - assertThat(e.getMessage(), containsString("The key format 'AVRO' is not currently supported.")); + assertThat(e.getMessage(), containsString("The key format 'PROTOBUF' is not currently supported.")); } @Test @@ -1493,14 +1493,14 @@ public void shouldThrowOnUnsupportedKeyFormatForCSAS() { () -> KsqlEngineTestUtil.execute( serviceContext, ksqlEngine, - "CREATE STREAM FOO WITH (KEY_FORMAT='AVRO') AS SELECT * FROM ORDERS;", + "CREATE STREAM FOO WITH (KEY_FORMAT='PROTOBUF') AS SELECT * FROM ORDERS;", KSQL_CONFIG, Collections.emptyMap() ) ); // Then: - assertThat(e, rawMessage(containsString("The key format 'AVRO' is not currently supported."))); + assertThat(e, rawMessage(containsString("The key format 'PROTOBUF' is not currently supported."))); } @Test diff --git a/ksqldb-engine/src/test/java/io/confluent/ksql/integration/EndToEndIntegrationTest.java b/ksqldb-engine/src/test/java/io/confluent/ksql/integration/EndToEndIntegrationTest.java index 465e9f6e2bc1..36e35e76b09b 100644 --- a/ksqldb-engine/src/test/java/io/confluent/ksql/integration/EndToEndIntegrationTest.java +++ b/ksqldb-engine/src/test/java/io/confluent/ksql/integration/EndToEndIntegrationTest.java @@ -124,10 +124,6 @@ public class EndToEndIntegrationTest { .withAdditionalConfig( KsqlConfig.SCHEMA_REGISTRY_URL_PROPERTY, "http://foo:8080") - .withAdditionalConfig( - KsqlConfig.KSQL_KEY_FORMAT_ENABLED, - true - ) .build(); @Rule diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_inference/6.2.0_1606796938952/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_inference/6.2.0_1606796938952/plan.json new file mode 100644 index 000000000000..0dd87b151f81 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_inference/6.2.0_1606796938952/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY BIGINT KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_inference/6.2.0_1606796938952/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_inference/6.2.0_1606796938952/spec.json new file mode 100644 index 000000000000..a2626a88ba25 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_inference/6.2.0_1606796938952/spec.json @@ -0,0 +1,149 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796938952, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "BIGINT - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 998877665544332211, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 998877665544332211, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "long", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 1, + "keySchema" : "long", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "long", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_inference/6.2.0_1606796938952/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_inference/6.2.0_1606796938952/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_inference/6.2.0_1606796938952/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_no_inference/6.2.0_1606796938914/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_no_inference/6.2.0_1606796938914/plan.json new file mode 100644 index 000000000000..648cb4258ce0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_no_inference/6.2.0_1606796938914/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K BIGINT KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` BIGINT KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_no_inference/6.2.0_1606796938914/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_no_inference/6.2.0_1606796938914/spec.json new file mode 100644 index 000000000000..1b26e7d2dacf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_no_inference/6.2.0_1606796938914/spec.json @@ -0,0 +1,149 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796938914, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "BIGINT - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 998877665544332211, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 998877665544332211, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "long", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K BIGINT KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "long", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "long", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_no_inference/6.2.0_1606796938914/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_no_inference/6.2.0_1606796938914/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BIGINT_-_key_-_no_inference/6.2.0_1606796938914/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_inference/6.2.0_1606796938829/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_inference/6.2.0_1606796938829/plan.json new file mode 100644 index 000000000000..cf1eb1cf862d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_inference/6.2.0_1606796938829/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY BOOLEAN KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_inference/6.2.0_1606796938829/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_inference/6.2.0_1606796938829/spec.json new file mode 100644 index 000000000000..1fbea6deacc6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_inference/6.2.0_1606796938829/spec.json @@ -0,0 +1,145 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796938829, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "BOOLEAN - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : true, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : null + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 1, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_inference/6.2.0_1606796938829/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_inference/6.2.0_1606796938829/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_inference/6.2.0_1606796938829/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796938782/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796938782/plan.json new file mode 100644 index 000000000000..63dfa2fd5c0c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796938782/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K BOOLEAN KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` BOOLEAN KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796938782/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796938782/spec.json new file mode 100644 index 000000000000..3c3bbfbbe3d2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796938782/spec.json @@ -0,0 +1,145 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796938782, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "BOOLEAN - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : true, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : null + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K BOOLEAN KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796938782/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796938782/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796938782/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_inference/6.2.0_1606796939198/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_inference/6.2.0_1606796939198/plan.json new file mode 100644 index 000000000000..b9b8effb972f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_inference/6.2.0_1606796939198/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY DECIMAL(4, 2) KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_inference/6.2.0_1606796939198/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_inference/6.2.0_1606796939198/spec.json new file mode 100644 index 000000000000..7c632e3f3b6a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_inference/6.2.0_1606796939198/spec.json @@ -0,0 +1,170 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796939198, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "DECIMAL - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 65.21, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 65.21, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "type" : "bytes", + "logicalType" : "decimal", + "precision" : 4, + "scale" : 2 + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 1, + "keySchema" : { + "type" : "bytes", + "scale" : 2, + "precision" : 4, + "connect.version" : 1, + "connect.parameters" : { + "scale" : "2", + "connect.decimal.precision" : "4" + }, + "connect.name" : "org.apache.kafka.connect.data.Decimal", + "logicalType" : "decimal" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : { + "type" : "bytes", + "scale" : 2, + "precision" : 4, + "logicalType" : "decimal" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_inference/6.2.0_1606796939198/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_inference/6.2.0_1606796939198/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_inference/6.2.0_1606796939198/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796939165/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796939165/plan.json new file mode 100644 index 000000000000..7e3de771758d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796939165/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K DECIMAL(4, 2) KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796939165/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796939165/spec.json new file mode 100644 index 000000000000..d1eee2b7d0e6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796939165/spec.json @@ -0,0 +1,175 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796939165, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "DECIMAL - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 65.21, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 65.21, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "type" : "bytes", + "scale" : 2, + "precision" : 4, + "connect.parameters" : { + "scale" : "2", + "connect.decimal.precision" : "4" + }, + "connect.name" : "org.apache.kafka.connect.data.Decimal", + "logicalType" : "decimal" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K DECIMAL(4,2) KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : { + "type" : "bytes", + "scale" : 2, + "precision" : 4, + "connect.version" : 1, + "connect.parameters" : { + "scale" : "2", + "connect.decimal.precision" : "4" + }, + "connect.name" : "org.apache.kafka.connect.data.Decimal", + "logicalType" : "decimal" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : { + "type" : "bytes", + "scale" : 2, + "precision" : 4, + "logicalType" : "decimal" + }, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796939165/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796939165/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796939165/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_inference/6.2.0_1606796939013/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_inference/6.2.0_1606796939013/plan.json new file mode 100644 index 000000000000..8c2321dd68c3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_inference/6.2.0_1606796939013/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY DOUBLE KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_inference/6.2.0_1606796939013/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_inference/6.2.0_1606796939013/spec.json new file mode 100644 index 000000000000..615f661c003f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_inference/6.2.0_1606796939013/spec.json @@ -0,0 +1,149 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796939013, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "DOUBLE - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 654.321, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 654.321, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "double", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 1, + "keySchema" : "double", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "double", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_inference/6.2.0_1606796939013/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_inference/6.2.0_1606796939013/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_inference/6.2.0_1606796939013/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796938983/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796938983/plan.json new file mode 100644 index 000000000000..79a79da5ea84 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796938983/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K DOUBLE KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` DOUBLE KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796938983/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796938983/spec.json new file mode 100644 index 000000000000..4a4a0f8421bb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796938983/spec.json @@ -0,0 +1,149 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796938983, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "DOUBLE - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 654.321, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 654.321, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "double", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K DOUBLE KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "double", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "double", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796938983/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796938983/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796938983/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_inference/6.2.0_1606796938884/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_inference/6.2.0_1606796938884/plan.json new file mode 100644 index 000000000000..61871663d9ed --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_inference/6.2.0_1606796938884/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY INTEGER KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_inference/6.2.0_1606796938884/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_inference/6.2.0_1606796938884/spec.json new file mode 100644 index 000000000000..c8a81513b6e1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_inference/6.2.0_1606796938884/spec.json @@ -0,0 +1,161 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796938884, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "INT - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 1, + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_inference/6.2.0_1606796938884/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_inference/6.2.0_1606796938884/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_inference/6.2.0_1606796938884/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796939253/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796939253/plan.json new file mode 100644 index 000000000000..7677a8f35628 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796939253/plan.json @@ -0,0 +1,149 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY INTEGER KEY, ROWVAL INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1, VALUE_SCHEMA_ID=1, WRAP_SINGLE_VALUE=false);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "ROWVAL AS ROWVAL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796939253/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796939253/spec.json new file mode 100644 index 000000000000..c9472b6535bf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796939253/spec.json @@ -0,0 +1,135 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796939253, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "INT - key - key and anonymous value inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 10, + "value" : 10 + }, { + "topic" : "input_topic", + "key" : 12, + "value" : null + }, { + "topic" : "input_topic", + "key" : null, + "value" : 10 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "ROWVAL" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : 12, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "ROWVAL" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "int", + "valueSchema" : "int", + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT WITH (kafka_topic='input_topic', format='AVRO', wrap_single_value=false);", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ "UNWRAP_SINGLES" ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "partitions" : 1, + "keySchema" : "int", + "valueSchema" : "int" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "ROWVAL", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796939253/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796939253/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796939253/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796939229/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796939229/plan.json new file mode 100644 index 000000000000..22416a80105f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796939229/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY INTEGER KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1, VALUE_SCHEMA_ID=2);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796939229/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796939229/spec.json new file mode 100644 index 000000000000..bab944f55542 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796939229/spec.json @@ -0,0 +1,156 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796939229, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "INT - key - key and value inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "most_recent_value_schema_at_SR", + "fields" : [ { + "name" : "FOO", + "type" : "int" + } ] + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 1, + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "most_recent_value_schema_at_SR", + "fields" : [ { + "name" : "FOO", + "type" : "int" + } ], + "connect.name" : "most_recent_value_schema_at_SR" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796939229/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796939229/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796939229/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_no_inference/6.2.0_1606796938855/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_no_inference/6.2.0_1606796938855/plan.json new file mode 100644 index 000000000000..89f9a33ad937 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_no_inference/6.2.0_1606796938855/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K INTEGER KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` INTEGER KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_no_inference/6.2.0_1606796938855/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_no_inference/6.2.0_1606796938855/spec.json new file mode 100644 index 000000000000..2bc3669eeede --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_no_inference/6.2.0_1606796938855/spec.json @@ -0,0 +1,161 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796938855, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "INT - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K INT KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_no_inference/6.2.0_1606796938855/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_no_inference/6.2.0_1606796938855/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_INT_-_key_-_no_inference/6.2.0_1606796938855/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_inference/6.2.0_1606796939104/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_inference/6.2.0_1606796939104/plan.json new file mode 100644 index 000000000000..a44ddac66bbf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_inference/6.2.0_1606796939104/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY STRING KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` STRING KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_inference/6.2.0_1606796939104/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_inference/6.2.0_1606796939104/spec.json new file mode 100644 index 000000000000..3186964655a3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_inference/6.2.0_1606796939104/spec.json @@ -0,0 +1,149 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796939104, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "STRING - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : "foo", + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "foo", + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "string", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 1, + "keySchema" : "string", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "string", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_inference/6.2.0_1606796939104/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_inference/6.2.0_1606796939104/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_inference/6.2.0_1606796939104/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_no_inference/6.2.0_1606796939050/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_no_inference/6.2.0_1606796939050/plan.json new file mode 100644 index 000000000000..99076d018a3f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_no_inference/6.2.0_1606796939050/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K STRING KEY, FOO INTEGER) WITH (FORMAT='AVRO', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` STRING KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_no_inference/6.2.0_1606796939050/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_no_inference/6.2.0_1606796939050/spec.json new file mode 100644 index 000000000000..fd34f8752d84 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_no_inference/6.2.0_1606796939050/spec.json @@ -0,0 +1,149 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796939050, + "path" : "query-validation-tests/avro.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "STRING - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : "foo", + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "foo", + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : "string", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K VARCHAR KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "string", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "string", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FOO", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_no_inference/6.2.0_1606796939050/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_no_inference/6.2.0_1606796939050/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/avro_-_STRING_-_key_-_no_inference/6.2.0_1606796939050/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1606799798169/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1606799798169/plan.json new file mode 100644 index 000000000000..01d93ddf1469 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1606799798169/plan.json @@ -0,0 +1,190 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID INTEGER KEY, VAL BOOLEAN) WITH (FORMAT='AVRO', KAFKA_TOPIC='test_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` INTEGER KEY, `VAL` BOOLEAN", + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n TEST.VAL VAL,\n COUNT(*) COUNT\nFROM TEST TEST\nGROUP BY TEST.VAL\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `VAL` BOOLEAN" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "VAL AS VAL", "ROWTIME AS ROWTIME" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "groupByExpressions" : [ "VAL" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "nonAggregateColumns" : [ "VAL", "ROWTIME" ], + "aggregationFunctions" : [ "COUNT(ROWTIME)" ] + }, + "keyColumnNames" : [ "VAL" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1606799798169/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1606799798169/spec.json new file mode 100644 index 000000000000..4aa3fca58381 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1606799798169/spec.json @@ -0,0 +1,228 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799798169, + "path" : "query-validation-tests/group-by.json", + "schemas" : { + "CTAS_OUTPUT_0.Aggregate.GroupBy" : { + "schema" : "`VAL` BOOLEAN KEY, `VAL` BOOLEAN, `ROWTIME` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` INTEGER KEY, `VAL` BOOLEAN", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`VAL` BOOLEAN KEY, `VAL` BOOLEAN, `ROWTIME` BIGINT, `KSQL_AGG_VARIABLE_0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "AVRO primitive key", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VAL" : true + } + }, { + "topic" : "test_topic", + "key" : 1, + "value" : { + "VAL" : false + } + }, { + "topic" : "test_topic", + "key" : 2, + "value" : { + "VAL" : true + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : false, + "value" : { + "COUNT" : 1 + } + }, { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "COUNT" : 2 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "ignored", + "fields" : [ { + "name" : "VAL", + "type" : "boolean" + } ] + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM TEST (ID INT KEY, VAL BOOLEAN) WITH (kafka_topic='test_topic', format='AVRO');", "CREATE TABLE OUTPUT AS SELECT VAL, COUNT() AS COUNT FROM TEST GROUP BY VAL;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`VAL` BOOLEAN KEY, `COUNT` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `VAL` BOOLEAN", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 1, + "keySchema" : "int", + "valueSchema" : { + "type" : "record", + "name" : "ignored", + "fields" : [ { + "name" : "VAL", + "type" : "boolean" + } ], + "connect.name" : "ignored" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VAL", + "type" : [ "null", "boolean" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "VAL", + "type" : [ "null", "boolean" ], + "default" : null + }, { + "name" : "ROWTIME", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "KSQL_AGG_VARIABLE_0", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "boolean", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "COUNT", + "type" : [ "null", "long" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1606799798169/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1606799798169/topology new file mode 100644 index 000000000000..99948fd4b5c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/group-by_-_AVRO_primitive_key/6.2.0_1606799798169/topology @@ -0,0 +1,40 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-FILTER-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-FILTER-0000000003 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> Aggregate-GroupBy-repartition-filter + <-- KSTREAM-FILTER-0000000003 + Processor: Aggregate-GroupBy-repartition-filter (stores: []) + --> Aggregate-GroupBy-repartition-sink + <-- Aggregate-GroupBy + Sink: Aggregate-GroupBy-repartition-sink (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy-repartition-filter + + Sub-topology: 1 + Source: Aggregate-GroupBy-repartition-source (topics: [Aggregate-GroupBy-repartition]) + --> KSTREAM-AGGREGATE-0000000005 + Processor: KSTREAM-AGGREGATE-0000000005 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-GroupBy-repartition-source + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000005 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000011 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000011 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000011 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_session-windowed_-_SR-enabled_key_format/6.2.0_1606799813835/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_session-windowed_-_SR-enabled_key_format/6.2.0_1606799813835/plan.json new file mode 100644 index 000000000000..9ca396ce82a5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_session-windowed_-_SR-enabled_key_format/6.2.0_1606799813835/plan.json @@ -0,0 +1,258 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='JSON_SR', VALUE_FORMAT='JSON', WINDOW_TYPE='SESSION');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "SESSION" + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, V BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='JSON_SR', VALUE_FORMAT='JSON', WINDOW_TYPE='SESSION');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "SESSION" + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ID S1_ID,\n S1.V S1_V,\n S2.V S2_V\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 1 MINUTES ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_V` BIGINT, `S2_V` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "SESSION" + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "windowedStreamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "SESSION" + }, + "sourceSchema" : "`ID` INTEGER KEY, `V` BIGINT" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V AS S1_V", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID", "WINDOWSTART AS S1_WINDOWSTART", "WINDOWEND AS S1_WINDOWEND" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "windowedStreamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "SESSION" + }, + "sourceSchema" : "`ID` INTEGER KEY, `V` BIGINT" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "V AS S2_V", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID", "WINDOWSTART AS S2_WINDOWSTART", "WINDOWEND AS S2_WINDOWEND" ] + }, + "beforeMillis" : 60.000000000, + "afterMillis" : 60.000000000, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_V AS S1_V", "S2_V AS S2_V" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_session-windowed_-_SR-enabled_key_format/6.2.0_1606799813835/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_session-windowed_-_SR-enabled_key_format/6.2.0_1606799813835/spec.json new file mode 100644 index 000000000000..4074478af650 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_session-windowed_-_SR-enabled_key_format/6.2.0_1606799813835/spec.json @@ -0,0 +1,336 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799813835, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S1_WINDOWSTART` BIGINT, `S1_WINDOWEND` BIGINT", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V` BIGINT, `S2_V` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S1_WINDOWSTART` BIGINT, `S1_WINDOWEND` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`S2_ID` INTEGER KEY, `S2_V` BIGINT, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER, `S2_WINDOWSTART` BIGINT, `S2_WINDOWEND` BIGINT", + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "matching session-windowed - SR-enabled key format", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "V" : 1 + }, + "timestamp" : 765, + "window" : { + "start" : 234, + "end" : 765, + "type" : "SESSION" + } + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "V" : 2 + }, + "timestamp" : 567, + "window" : { + "start" : 234, + "end" : 567, + "type" : "SESSION" + } + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "V" : 3 + }, + "timestamp" : 765, + "window" : { + "start" : 234, + "end" : 765, + "type" : "SESSION" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "S1_V" : 1, + "S2_V" : 3 + }, + "timestamp" : 765, + "window" : { + "start" : 234, + "end" : 765, + "type" : "SESSION" + } + } ], + "topics" : [ { + "name" : "right_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V bigint) WITH (kafka_topic='left_topic', key_format='JSON_SR', value_format='JSON', WINDOW_TYPE='SESSION');", "CREATE STREAM S2 (ID INT KEY, V bigint) WITH (kafka_topic='right_topic', key_format='JSON_SR', value_format='JSON', WINDOW_TYPE='SESSION');", "CREATE STREAM OUTPUT as SELECT S1.ID, S1.V, S2.V FROM S1 JOIN S2 WITHIN 1 MINUTE ON S1.ID = S2.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S1_ID` INTEGER KEY, `S1_V` BIGINT, `S2_V` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "windowType" : "SESSION" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "windowType" : "SESSION" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "windowType" : "SESSION" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "SESSION" + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_session-windowed_-_SR-enabled_key_format/6.2.0_1606799813835/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_session-windowed_-_SR-enabled_key_format/6.2.0_1606799813835/topology new file mode 100644 index 000000000000..e6d9bd120b1e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_session-windowed_-_SR-enabled_key_format/6.2.0_1606799813835/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_time-windowed_-_SR-enabled_key_format/6.2.0_1606799813939/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_time-windowed_-_SR-enabled_key_format/6.2.0_1606799813939/plan.json new file mode 100644 index 000000000000..f95da66aea26 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_time-windowed_-_SR-enabled_key_format/6.2.0_1606799813939/plan.json @@ -0,0 +1,263 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='JSON_SR', VALUE_FORMAT='JSON', WINDOW_SIZE='5 SECONDS', WINDOW_TYPE='Hopping');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, V BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='JSON_SR', VALUE_FORMAT='JSON', WINDOW_SIZE='2 SECOND', WINDOW_TYPE='Tumbling');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "TUMBLING", + "size" : 2.000000000 + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n *,\n S1.ROWTIME S1_ROWTIME,\n S2.ROWTIME S2_ROWTIME\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 1 MINUTES ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `S1_WINDOWSTART` BIGINT, `S1_WINDOWEND` BIGINT, `S1_V` BIGINT, `S2_ID` INTEGER, `S2_WINDOWSTART` BIGINT, `S2_WINDOWEND` BIGINT, `S2_V` BIGINT, `S1_ROWTIME` BIGINT, `S2_ROWTIME` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "windowedStreamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + }, + "sourceSchema" : "`ID` INTEGER KEY, `V` BIGINT" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V AS S1_V", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID", "WINDOWSTART AS S1_WINDOWSTART", "WINDOWEND AS S1_WINDOWEND" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "windowedStreamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "windowInfo" : { + "type" : "TUMBLING", + "size" : 2.000000000 + }, + "sourceSchema" : "`ID` INTEGER KEY, `V` BIGINT" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "V AS S2_V", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID", "WINDOWSTART AS S2_WINDOWSTART", "WINDOWEND AS S2_WINDOWEND" ] + }, + "beforeMillis" : 60.000000000, + "afterMillis" : 60.000000000, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_WINDOWSTART AS S1_WINDOWSTART", "S1_WINDOWEND AS S1_WINDOWEND", "S1_V AS S1_V", "S2_ID AS S2_ID", "S2_WINDOWSTART AS S2_WINDOWSTART", "S2_WINDOWEND AS S2_WINDOWEND", "S2_V AS S2_V", "S1_ROWTIME AS S1_ROWTIME", "S2_ROWTIME AS S2_ROWTIME" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_time-windowed_-_SR-enabled_key_format/6.2.0_1606799813939/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_time-windowed_-_SR-enabled_key_format/6.2.0_1606799813939/spec.json new file mode 100644 index 000000000000..c72fcbd0fc39 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_time-windowed_-_SR-enabled_key_format/6.2.0_1606799813939/spec.json @@ -0,0 +1,401 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799813939, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "TUMBLING", + "size" : 2.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S1_WINDOWSTART` BIGINT, `S1_WINDOWEND` BIGINT", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_WINDOWSTART` BIGINT, `S1_WINDOWEND` BIGINT, `S1_V` BIGINT, `S2_ID` INTEGER, `S2_WINDOWSTART` BIGINT, `S2_WINDOWEND` BIGINT, `S2_V` BIGINT, `S1_ROWTIME` BIGINT, `S2_ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S1_WINDOWSTART` BIGINT, `S1_WINDOWEND` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`S2_ID` INTEGER KEY, `S2_V` BIGINT, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER, `S2_WINDOWSTART` BIGINT, `S2_WINDOWEND` BIGINT", + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "matching time-windowed - SR-enabled key format", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "V" : 1 + }, + "timestamp" : 0, + "window" : { + "start" : 0, + "end" : 5000, + "type" : "TIME" + } + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "V" : 2 + }, + "timestamp" : 1000, + "window" : { + "start" : 1000, + "end" : 6000, + "type" : "TIME" + } + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "V" : 3 + }, + "timestamp" : 2000, + "window" : { + "start" : 2000, + "end" : 7000, + "type" : "TIME" + } + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "V" : 4 + }, + "timestamp" : 0, + "window" : { + "start" : 0, + "end" : 2000, + "type" : "TIME" + } + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "V" : 5 + }, + "timestamp" : 2000, + "window" : { + "start" : 2000, + "end" : 4000, + "type" : "TIME" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "S1_ROWTIME" : 0, + "S1_WINDOWSTART" : 0, + "S1_WINDOWEND" : 5000, + "S1_V" : 1, + "S2_ROWTIME" : 0, + "S2_WINDOWSTART" : 0, + "S2_WINDOWEND" : 2000, + "S2_ID" : 1, + "S2_V" : 4 + }, + "timestamp" : 0, + "window" : { + "start" : 0, + "end" : 5000, + "type" : "TIME" + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "S1_ROWTIME" : 2000, + "S1_WINDOWSTART" : 2000, + "S1_WINDOWEND" : 7000, + "S1_V" : 3, + "S2_ROWTIME" : 2000, + "S2_WINDOWSTART" : 2000, + "S2_WINDOWEND" : 4000, + "S2_ID" : 1, + "S2_V" : 5 + }, + "timestamp" : 2000, + "window" : { + "start" : 2000, + "end" : 7000, + "type" : "TIME" + } + } ], + "topics" : [ { + "name" : "right_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V bigint) WITH (kafka_topic='left_topic', key_format='JSON_SR', value_format='JSON', WINDOW_TYPE='Hopping', WINDOW_SIZE='5 SECONDS');", "CREATE STREAM S2 (ID INT KEY, V bigint) WITH (kafka_topic='right_topic', key_format='JSON_SR', value_format='JSON', WINDOW_TYPE='Tumbling', WINDOW_SIZE='2 SECOND');", "CREATE STREAM OUTPUT as SELECT *, S1.ROWTIME, S2.ROWTIME FROM S1 JOIN S2 WITHIN 1 MINUTE ON S1.ID = S2.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S1_ID` INTEGER KEY, `S1_WINDOWSTART` BIGINT, `S1_WINDOWEND` BIGINT, `S1_V` BIGINT, `S2_ID` INTEGER, `S2_WINDOWSTART` BIGINT, `S2_WINDOWEND` BIGINT, `S2_V` BIGINT, `S1_ROWTIME` BIGINT, `S2_ROWTIME` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "windowType" : "HOPPING", + "windowSize" : 5000 + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "windowType" : "HOPPING", + "windowSize" : 5000 + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "windowType" : "TUMBLING", + "windowSize" : 2000 + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "HOPPING", + "size" : 5.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ], + "windowInfo" : { + "type" : "TUMBLING", + "size" : 2.000000000 + } + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_time-windowed_-_SR-enabled_key_format/6.2.0_1606799813939/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_time-windowed_-_SR-enabled_key_format/6.2.0_1606799813939/topology new file mode 100644 index 000000000000..e6d9bd120b1e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_matching_time-windowed_-_SR-enabled_key_format/6.2.0_1606799813939/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_and_value_format_-_with_inference_-_different_key_schemas/6.2.0_1606799813622/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_and_value_format_-_with_inference_-_different_key_schemas/6.2.0_1606799813622/plan.json new file mode 100644 index 000000000000..21cf7ca9ade5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_and_value_format_-_with_inference_-_different_key_schemas/6.2.0_1606799813622/plan.json @@ -0,0 +1,243 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY INTEGER KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='s1', KEY_SCHEMA_ID=1, VALUE_SCHEMA_ID=2);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ROWKEY INTEGER KEY, VAL STRING) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='s2', KEY_SCHEMA_ID=3, VALUE_SCHEMA_ID=4);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ROWKEY S1_ROWKEY,\n S2.VAL VAL\nFROM S1 S1\nLEFT OUTER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.ROWKEY = S2.ROWKEY))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ROWKEY` INTEGER KEY, `VAL` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER" + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "S1_ROWKEY" ], + "selectExpressions" : [ "FOO AS S1_FOO", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `VAL` STRING" + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "S2_ROWKEY" ], + "selectExpressions" : [ "VAL AS S2_VAL", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S1_ROWKEY" + }, + "keyColumnNames" : [ "S1_ROWKEY" ], + "selectExpressions" : [ "S2_VAL AS VAL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_and_value_format_-_with_inference_-_different_key_schemas/6.2.0_1606799813622/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_and_value_format_-_with_inference_-_different_key_schemas/6.2.0_1606799813622/spec.json new file mode 100644 index 000000000000..de56251486c8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_and_value_format_-_with_inference_-_different_key_schemas/6.2.0_1606799813622/spec.json @@ -0,0 +1,464 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799813622, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S1_ROWKEY` INTEGER KEY, `S1_FOO` INTEGER, `S1_ROWTIME` BIGINT, `S1_ROWKEY` INTEGER", + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S1_ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S1_ROWKEY` INTEGER KEY, `S1_FOO` INTEGER, `S1_ROWTIME` BIGINT, `S1_ROWKEY` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`S2_ROWKEY` INTEGER KEY, `S2_VAL` STRING, `S2_ROWTIME` BIGINT, `S2_ROWKEY` INTEGER", + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "stream-stream key-to-key - SR-enabled key and value format - with inference - different key schemas", + "inputs" : [ { + "topic" : "s1", + "key" : 10, + "value" : { + "foo" : 22 + } + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "VAL" : "hello" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "VAL" : null + } + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "VAL" : "hello" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "keySchema" : { + "type" : "integer", + "connect.type" : "int32" + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + }, { + "name" : "s2", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "VAL" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "string", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM S1 WITH (kafka_topic='s1', format='JSON_SR');", "CREATE STREAM S2 WITH (kafka_topic='s2', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT S1.ROWKEY, S2.VAL FROM S1 LEFT OUTER JOIN S2 WITHIN 10 SECONDS ON S1.ROWKEY = S2.ROWKEY;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S1_ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "S1_ROWTIME" : { + "connect.index" : 1, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int64" + } ] + }, + "S1_FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "S1_ROWKEY" : { + "connect.index" : 2, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "s2", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "VAL" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "string", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "s1", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 1, + "keySchema" : { + "type" : "integer", + "connect.type" : "int32" + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "S2_ROWTIME" : { + "connect.index" : 1, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int64" + } ] + }, + "S2_VAL" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "string" + } ] + }, + "S2_ROWKEY" : { + "connect.index" : 2, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "S1_ROWTIME" : { + "connect.index" : 1, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int64" + } ] + }, + "S1_FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "S1_ROWKEY" : { + "connect.index" : 2, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "VAL" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "string" + } ] + } + } + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "S2_ROWTIME" : { + "connect.index" : 1, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int64" + } ] + }, + "S2_VAL" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "string" + } ] + }, + "S2_ROWKEY" : { + "connect.index" : 2, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_and_value_format_-_with_inference_-_different_key_schemas/6.2.0_1606799813622/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_and_value_format_-_with_inference_-_different_key_schemas/6.2.0_1606799813622/topology new file mode 100644 index 000000000000..a1f74dde8081 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_and_value_format_-_with_inference_-_different_key_schemas/6.2.0_1606799813622/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-outer-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-outer-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812675/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812675/plan.json new file mode 100644 index 000000000000..e54b80af764b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812675/plan.json @@ -0,0 +1,243 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, FOO INTEGER) WITH (KAFKA_TOPIC='s1', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `FOO` INTEGER", + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, VAL STRING) WITH (KAFKA_TOPIC='s2', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `VAL` STRING", + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ID S1_ID,\n S2.VAL VAL\nFROM S1 S1\nLEFT OUTER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ID` INTEGER KEY, `VAL` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `FOO` INTEGER" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "FOO AS S1_FOO", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `VAL` STRING" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "VAL AS S2_VAL", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S2_VAL AS VAL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812675/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812675/spec.json new file mode 100644 index 000000000000..b9263acc818f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812675/spec.json @@ -0,0 +1,212 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799812675, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_FOO` INTEGER, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S1_ID` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_FOO` INTEGER, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`S2_ID` INTEGER KEY, `S2_VAL` STRING, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "stream-stream key-to-key - SR-enabled key format", + "inputs" : [ { + "topic" : "s1", + "key" : 10, + "value" : { + "foo" : 22 + } + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "VAL" : "hello" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "VAL" : null + } + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "VAL" : "hello" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s2", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, FOO INT) WITH (kafka_topic='s1', key_format='AVRO', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, VAL STRING) WITH (kafka_topic='s2', key_format='AVRO', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT S1.ID, S2.VAL FROM S1 LEFT OUTER JOIN S2 WITHIN 10 SECONDS ON S1.ID = S2.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S1_ID` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "s2", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "s1", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812675/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812675/topology new file mode 100644 index 000000000000..a1f74dde8081 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812675/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-outer-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-outer-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference/6.2.0_1606799813157/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference/6.2.0_1606799813157/plan.json new file mode 100644 index 000000000000..99a49a2e826e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference/6.2.0_1606799813157/plan.json @@ -0,0 +1,243 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY INTEGER KEY, FOO INTEGER) WITH (KAFKA_TOPIC='s1', KEY_FORMAT='JSON_SR', KEY_SCHEMA_ID=1, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ROWKEY INTEGER KEY, VAL STRING) WITH (KAFKA_TOPIC='s2', KEY_FORMAT='JSON_SR', KEY_SCHEMA_ID=1, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ROWKEY S1_ROWKEY,\n S2.VAL VAL\nFROM S1 S1\nLEFT OUTER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.ROWKEY = S2.ROWKEY))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ROWKEY` INTEGER KEY, `VAL` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER" + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "S1_ROWKEY" ], + "selectExpressions" : [ "FOO AS S1_FOO", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `VAL` STRING" + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "S2_ROWKEY" ], + "selectExpressions" : [ "VAL AS S2_VAL", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S1_ROWKEY" + }, + "keyColumnNames" : [ "S1_ROWKEY" ], + "selectExpressions" : [ "S2_VAL AS VAL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference/6.2.0_1606799813157/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference/6.2.0_1606799813157/spec.json new file mode 100644 index 000000000000..934b356763ff --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference/6.2.0_1606799813157/spec.json @@ -0,0 +1,275 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799813157, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S1_ROWKEY` INTEGER KEY, `S1_FOO` INTEGER, `S1_ROWTIME` BIGINT, `S1_ROWKEY` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S1_ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S1_ROWKEY` INTEGER KEY, `S1_FOO` INTEGER, `S1_ROWTIME` BIGINT, `S1_ROWKEY` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`S2_ROWKEY` INTEGER KEY, `S2_VAL` STRING, `S2_ROWTIME` BIGINT, `S2_ROWKEY` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "stream-stream key-to-key - SR-enabled key format - with inference", + "inputs" : [ { + "topic" : "s1", + "key" : 10, + "value" : { + "foo" : 22 + } + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "VAL" : "hello" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "VAL" : null + } + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "VAL" : "hello" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + }, { + "name" : "s2", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM S1 (FOO INT) WITH (kafka_topic='s1', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM S2 (VAL STRING) WITH (kafka_topic='s2', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT S1.ROWKEY, S2.VAL FROM S1 LEFT OUTER JOIN S2 WITHIN 10 SECONDS ON S1.ROWKEY = S2.ROWKEY;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S1_ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "s2", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "s1", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference/6.2.0_1606799813157/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference/6.2.0_1606799813157/topology new file mode 100644 index 000000000000..a1f74dde8081 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference/6.2.0_1606799813157/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-outer-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-outer-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813258/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813258/plan.json new file mode 100644 index 000000000000..a77e6de585f6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813258/plan.json @@ -0,0 +1,243 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY INTEGER KEY, FOO INTEGER) WITH (KAFKA_TOPIC='s1', KEY_FORMAT='JSON_SR', KEY_SCHEMA_ID=1, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ROWKEY INTEGER KEY, VAL STRING) WITH (KAFKA_TOPIC='s2', KEY_FORMAT='JSON_SR', KEY_SCHEMA_ID=2, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S1.ROWKEY S1_ROWKEY,\n S2.VAL VAL\nFROM S1 S1\nLEFT OUTER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.ROWKEY = S2.ROWKEY))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_ROWKEY` INTEGER KEY, `VAL` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER" + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "S1_ROWKEY" ], + "selectExpressions" : [ "FOO AS S1_FOO", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `VAL` STRING" + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "S2_ROWKEY" ], + "selectExpressions" : [ "VAL AS S2_VAL", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S1_ROWKEY" + }, + "keyColumnNames" : [ "S1_ROWKEY" ], + "selectExpressions" : [ "S2_VAL AS VAL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813258/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813258/spec.json new file mode 100644 index 000000000000..2bc712ae7fa6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813258/spec.json @@ -0,0 +1,267 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799813258, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S1_ROWKEY` INTEGER KEY, `S1_FOO` INTEGER, `S1_ROWTIME` BIGINT, `S1_ROWKEY` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S1_ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S1_ROWKEY` INTEGER KEY, `S1_FOO` INTEGER, `S1_ROWTIME` BIGINT, `S1_ROWKEY` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`S2_ROWKEY` INTEGER KEY, `S2_VAL` STRING, `S2_ROWTIME` BIGINT, `S2_ROWKEY` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "stream-stream key-to-key - SR-enabled key format - with inference - different schemas", + "inputs" : [ { + "topic" : "s1", + "key" : 10, + "value" : { + "foo" : 22 + } + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "VAL" : "hello" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "VAL" : null + } + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "VAL" : "hello" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "keySchema" : { + "type" : "integer", + "connect.type" : "int32" + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + }, { + "name" : "s2", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM S1 (FOO INT) WITH (kafka_topic='s1', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM S2 (VAL STRING) WITH (kafka_topic='s2', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT S1.ROWKEY, S2.VAL FROM S1 LEFT OUTER JOIN S2 WITHIN 10 SECONDS ON S1.ROWKEY = S2.ROWKEY;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S1_ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "s2", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "s1", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 1, + "keySchema" : { + "type" : "integer", + "connect.type" : "int32" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813258/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813258/topology new file mode 100644 index 000000000000..a1f74dde8081 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813258/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-outer-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-outer-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas_-_outer_join/6.2.0_1606799813725/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas_-_outer_join/6.2.0_1606799813725/plan.json new file mode 100644 index 000000000000..2b7f35f8362d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas_-_outer_join/6.2.0_1606799813725/plan.json @@ -0,0 +1,243 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY INTEGER KEY, FOO INTEGER) WITH (KAFKA_TOPIC='s1', KEY_FORMAT='JSON_SR', KEY_SCHEMA_ID=1, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ROWKEY INTEGER KEY, VAL STRING) WITH (KAFKA_TOPIC='s2', KEY_FORMAT='JSON_SR', KEY_SCHEMA_ID=2, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n ROWKEY_1 ROWKEY_1,\n AS_VALUE(S1.ROWKEY) S1_KEY,\n AS_VALUE(S2.ROWKEY) S2_KEY,\n S2.VAL VAL\nFROM S1 S1\nFULL OUTER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.ROWKEY = S2.ROWKEY))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY_1` INTEGER KEY, `S1_KEY` INTEGER, `S2_KEY` INTEGER, `VAL` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER" + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "S1_ROWKEY" ], + "selectExpressions" : [ "FOO AS S1_FOO", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `VAL` STRING" + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "S2_ROWKEY" ], + "selectExpressions" : [ "VAL AS S2_VAL", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "ROWKEY_1" + }, + "keyColumnNames" : [ "ROWKEY_1" ], + "selectExpressions" : [ "AS_VALUE(S1_ROWKEY) AS S1_KEY", "AS_VALUE(S2_ROWKEY) AS S2_KEY", "S2_VAL AS VAL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas_-_outer_join/6.2.0_1606799813725/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas_-_outer_join/6.2.0_1606799813725/spec.json new file mode 100644 index 000000000000..124516f32436 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas_-_outer_join/6.2.0_1606799813725/spec.json @@ -0,0 +1,271 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799813725, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S1_ROWKEY` INTEGER KEY, `S1_FOO` INTEGER, `S1_ROWTIME` BIGINT, `S1_ROWKEY` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY_1` INTEGER KEY, `S1_KEY` INTEGER, `S2_KEY` INTEGER, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S1_ROWKEY` INTEGER KEY, `S1_FOO` INTEGER, `S1_ROWTIME` BIGINT, `S1_ROWKEY` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`S2_ROWKEY` INTEGER KEY, `S2_VAL` STRING, `S2_ROWTIME` BIGINT, `S2_ROWKEY` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "stream-stream key-to-key - SR-enabled key format - with inference - different schemas - outer join", + "inputs" : [ { + "topic" : "s1", + "key" : 10, + "value" : { + "foo" : 22 + } + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "VAL" : "hello" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "S1_KEY" : 10, + "S2_KEY" : null, + "VAL" : null + } + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "S1_KEY" : 10, + "S2_KEY" : 10, + "VAL" : "hello" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "keySchema" : { + "type" : "integer", + "connect.type" : "int32" + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + }, { + "name" : "s2", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM S1 (FOO INT) WITH (kafka_topic='s1', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM S2 (VAL STRING) WITH (kafka_topic='s2', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT ROWKEY_1, AS_VALUE(S1.ROWKEY) AS S1_KEY, AS_VALUE(S2.ROWKEY) AS S2_KEY, S2.VAL FROM S1 FULL OUTER JOIN S2 WITHIN 10 SECONDS ON S1.ROWKEY = S2.ROWKEY;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY_1` INTEGER KEY, `S1_KEY` INTEGER, `S2_KEY` INTEGER, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "s2", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "s1", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 1, + "keySchema" : { + "type" : "integer", + "connect.type" : "int32" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTERTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas_-_outer_join/6.2.0_1606799813725/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas_-_outer_join/6.2.0_1606799813725/topology new file mode 100644 index 000000000000..dbeb7381f75e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-stream_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas_-_outer_join/6.2.0_1606799813725/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-outer-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-OUTERTHIS-0000000016-store]) + --> Join-outer-this-join + <-- Join-left-repartition-source + Processor: Join-outer-other-join (stores: [KSTREAM-OUTERTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-outer-this-join (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-outer-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812739/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812739/plan.json new file mode 100644 index 000000000000..45109c536f1e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812739/plan.json @@ -0,0 +1,242 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T (ID INTEGER PRIMARY KEY, VAL STRING) WITH (KAFKA_TOPIC='t', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T", + "schema" : "`ID` INTEGER KEY, `VAL` STRING", + "topicName" : "t", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S (ID INTEGER KEY, FOO INTEGER) WITH (KAFKA_TOPIC='s', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S", + "schema" : "`ID` INTEGER KEY, `FOO` INTEGER", + "topicName" : "s", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S.ID S_ID,\n T.VAL VAL\nFROM S S\nINNER JOIN T T ON ((S.ID = T.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S_ID` INTEGER KEY, `VAL` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S", "T" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `FOO` INTEGER" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S_ID" ], + "selectExpressions" : [ "FOO AS S_FOO", "ROWTIME AS S_ROWTIME", "ID AS S_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `VAL` STRING", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "VAL AS T_VAL", "ROWTIME AS T_ROWTIME", "ID AS T_ID" ] + }, + "keyColName" : "S_ID" + }, + "keyColumnNames" : [ "S_ID" ], + "selectExpressions" : [ "T_VAL AS VAL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812739/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812739/spec.json new file mode 100644 index 000000000000..a18007b7e104 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812739/spec.json @@ -0,0 +1,210 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799812739, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S_ID` INTEGER KEY, `S_FOO` INTEGER, `S_ROWTIME` BIGINT, `S_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S_ID` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.RightSourceKeyed.Materialize" : { + "schema" : "`ID` INTEGER KEY, `VAL` STRING, `ROWTIME` BIGINT, `ID` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S_ID` INTEGER KEY, `S_FOO` INTEGER, `S_ROWTIME` BIGINT, `S_ID` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + } + } + }, + "testCase" : { + "name" : "stream-table key-to-key - SR-enabled key format", + "inputs" : [ { + "topic" : "t", + "key" : 10, + "value" : { + "VAL" : "hello" + } + }, { + "topic" : "s", + "key" : 10, + "value" : { + "foo" : 22 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "VAL" : "hello" + } + } ], + "topics" : [ { + "name" : "s", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "t", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE T (ID INT PRIMARY KEY, VAL STRING) WITH (kafka_topic='t', key_format='AVRO', value_format='JSON');", "CREATE STREAM S (ID INT KEY, FOO INT) WITH (kafka_topic='s', key_format='AVRO', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT S.ID, VAL FROM S JOIN T ON S.ID = T.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S_ID` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "t", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-RightSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "s", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-RightSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812739/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812739/topology new file mode 100644 index 000000000000..4b6090f3aa0a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format/6.2.0_1606799812739/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [t]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> KTABLE-TOSTREAM-0000000005 + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TOSTREAM-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: RightSourceKeyed-SelectKey-Mapper (stores: []) + --> RightSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000005 + Processor: RightSourceKeyed-SelectKey-repartition-filter (stores: []) + --> RightSourceKeyed-SelectKey-repartition-sink + <-- RightSourceKeyed-SelectKey-Mapper + Sink: RightSourceKeyed-SelectKey-repartition-sink (topic: RightSourceKeyed-SelectKey-repartition) + <-- RightSourceKeyed-SelectKey-repartition-filter + + Sub-topology: 1 + Source: Join-repartition-source (topics: [Join-repartition]) + --> Join + Processor: Join (stores: [RightSourceKeyed-Materialize]) + --> Project + <-- Join-repartition-source + Source: RightSourceKeyed-SelectKey-repartition-source (topics: [RightSourceKeyed-SelectKey-repartition]) + --> RightSourceKeyed-SelectKey + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000021 + <-- Join + Processor: RightSourceKeyed-SelectKey (stores: [RightSourceKeyed-Materialize]) + --> PrependAliasRight + <-- RightSourceKeyed-SelectKey-repartition-source + Sink: KSTREAM-SINK-0000000021 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- RightSourceKeyed-SelectKey + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000012 (topics: [s]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000012 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: PrependAliasLeft (stores: []) + --> Join-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-repartition-filter (stores: []) + --> Join-repartition-sink + <-- PrependAliasLeft + Sink: Join-repartition-sink (topic: Join-repartition) + <-- Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813352/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813352/plan.json new file mode 100644 index 000000000000..2083a2e6e3ed --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813352/plan.json @@ -0,0 +1,242 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T (ROWKEY INTEGER PRIMARY KEY, VAL STRING) WITH (KAFKA_TOPIC='t', KEY_FORMAT='JSON_SR', KEY_SCHEMA_ID=2, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T", + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "topicName" : "t", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S (ROWKEY INTEGER KEY, FOO INTEGER) WITH (KAFKA_TOPIC='s', KEY_FORMAT='JSON_SR', KEY_SCHEMA_ID=1, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "s", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S.ROWKEY S_ROWKEY,\n T.VAL VAL\nFROM S S\nINNER JOIN T T ON ((S.ROWKEY = T.ROWKEY))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S_ROWKEY` INTEGER KEY, `VAL` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S", "T" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER" + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "S_ROWKEY" ], + "selectExpressions" : [ "FOO AS S_FOO", "ROWTIME AS S_ROWTIME", "ROWKEY AS S_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "T_ROWKEY" ], + "selectExpressions" : [ "VAL AS T_VAL", "ROWTIME AS T_ROWTIME", "ROWKEY AS T_ROWKEY" ] + }, + "keyColName" : "S_ROWKEY" + }, + "keyColumnNames" : [ "S_ROWKEY" ], + "selectExpressions" : [ "T_VAL AS VAL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813352/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813352/spec.json new file mode 100644 index 000000000000..87e4cab7cd50 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813352/spec.json @@ -0,0 +1,265 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799813352, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S_ROWKEY` INTEGER KEY, `S_FOO` INTEGER, `S_ROWTIME` BIGINT, `S_ROWKEY` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S_ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.RightSourceKeyed.Materialize" : { + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING, `ROWTIME` BIGINT, `ROWKEY` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S_ROWKEY` INTEGER KEY, `S_FOO` INTEGER, `S_ROWTIME` BIGINT, `S_ROWKEY` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + } + }, + "testCase" : { + "name" : "stream-table key-to-key - SR-enabled key format - with inference - different schemas", + "inputs" : [ { + "topic" : "t", + "key" : 10, + "value" : { + "VAL" : "hello" + } + }, { + "topic" : "s", + "key" : 10, + "value" : { + "foo" : 22 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "VAL" : "hello" + } + } ], + "topics" : [ { + "name" : "s", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + }, { + "name" : "t", + "keySchema" : { + "type" : "integer", + "connect.type" : "int32" + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE T (VAL STRING) WITH (kafka_topic='t', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM S (FOO INT) WITH (kafka_topic='s', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT S.ROWKEY, VAL FROM S JOIN T ON S.ROWKEY = T.ROWKEY;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S_ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T", + "type" : "TABLE", + "schema" : "`ROWKEY` INTEGER KEY, `VAL` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "t", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 1, + "keySchema" : { + "type" : "integer", + "connect.type" : "int32" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-RightSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "s", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-RightSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813352/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813352/topology new file mode 100644 index 000000000000..4b6090f3aa0a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream-table_key-to-key_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813352/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [t]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> KTABLE-TOSTREAM-0000000005 + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TOSTREAM-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: RightSourceKeyed-SelectKey-Mapper (stores: []) + --> RightSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000005 + Processor: RightSourceKeyed-SelectKey-repartition-filter (stores: []) + --> RightSourceKeyed-SelectKey-repartition-sink + <-- RightSourceKeyed-SelectKey-Mapper + Sink: RightSourceKeyed-SelectKey-repartition-sink (topic: RightSourceKeyed-SelectKey-repartition) + <-- RightSourceKeyed-SelectKey-repartition-filter + + Sub-topology: 1 + Source: Join-repartition-source (topics: [Join-repartition]) + --> Join + Processor: Join (stores: [RightSourceKeyed-Materialize]) + --> Project + <-- Join-repartition-source + Source: RightSourceKeyed-SelectKey-repartition-source (topics: [RightSourceKeyed-SelectKey-repartition]) + --> RightSourceKeyed-SelectKey + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000021 + <-- Join + Processor: RightSourceKeyed-SelectKey (stores: [RightSourceKeyed-Materialize]) + --> PrependAliasRight + <-- RightSourceKeyed-SelectKey-repartition-source + Sink: KSTREAM-SINK-0000000021 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- RightSourceKeyed-SelectKey + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000012 (topics: [s]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000012 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: PrependAliasLeft (stores: []) + --> Join-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-repartition-filter (stores: []) + --> Join-repartition-sink + <-- PrependAliasLeft + Sink: Join-repartition-sink (topic: Join-repartition) + <-- Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format/6.2.0_1606799812850/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format/6.2.0_1606799812850/plan.json new file mode 100644 index 000000000000..c6a48bf38fb8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format/6.2.0_1606799812850/plan.json @@ -0,0 +1,243 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='t1', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, F1 STRING) WITH (KAFKA_TOPIC='t2', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` BIGINT KEY, `F1` STRING", + "topicName" : "t2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T1.ID T1_ID,\n T1.NAME NAME,\n T2.F1 F1\nFROM T1 T1\nLEFT OUTER JOIN T2 T2 ON ((T1.ID = T2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T1_ID` BIGINT KEY, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "NAME AS T1_NAME", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "F1 AS T2_F1", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T1_ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "T1_NAME AS NAME", "T2_F1 AS F1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format/6.2.0_1606799812850/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format/6.2.0_1606799812850/spec.json new file mode 100644 index 000000000000..6024a0183ec6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format/6.2.0_1606799812850/spec.json @@ -0,0 +1,235 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799812850, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T1_ID` BIGINT KEY, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.RightSourceKeyed.Materialize" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `ROWTIME` BIGINT, `ID` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.LeftSourceKeyed.Materialize" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `ROWTIME` BIGINT, `ID` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "table-table - SR-enabled key format", + "inputs" : [ { + "topic" : "t1", + "key" : 0, + "value" : { + "NAME" : "zero" + } + }, { + "topic" : "t2", + "key" : 0, + "value" : { + "F1" : "blah" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "F1" : null + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "F1" : "blah" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "t1", + "keySchema" : "long", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "t2", + "keySchema" : "long", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, NAME varchar) WITH (kafka_topic='t1', key_format='AVRO', value_format='JSON');", "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, F1 varchar) WITH (kafka_topic='t2', key_format='AVRO', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT T1.id, name, f1 FROM T1 left join T2 on T1.id = T2.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`T1_ID` BIGINT KEY, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T1", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-LeftSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-RightSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-LeftSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-RightSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "t2", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "long" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "long" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "t1", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "long" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format/6.2.0_1606799812850/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format/6.2.0_1606799812850/topology new file mode 100644 index 000000000000..0209cbd53ffd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format/6.2.0_1606799812850/topology @@ -0,0 +1,84 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [t1]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> KTABLE-TOSTREAM-0000000005 + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TOSTREAM-0000000005 (stores: []) + --> LeftSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: LeftSourceKeyed-SelectKey-Mapper (stores: []) + --> LeftSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000005 + Processor: LeftSourceKeyed-SelectKey-repartition-filter (stores: []) + --> LeftSourceKeyed-SelectKey-repartition-sink + <-- LeftSourceKeyed-SelectKey-Mapper + Sink: LeftSourceKeyed-SelectKey-repartition-sink (topic: LeftSourceKeyed-SelectKey-repartition) + <-- LeftSourceKeyed-SelectKey-repartition-filter + + Sub-topology: 1 + Source: LeftSourceKeyed-SelectKey-repartition-source (topics: [LeftSourceKeyed-SelectKey-repartition]) + --> LeftSourceKeyed-SelectKey + Source: RightSourceKeyed-SelectKey-repartition-source (topics: [RightSourceKeyed-SelectKey-repartition]) + --> RightSourceKeyed-SelectKey + Processor: LeftSourceKeyed-SelectKey (stores: [LeftSourceKeyed-Materialize]) + --> PrependAliasLeft + <-- LeftSourceKeyed-SelectKey-repartition-source + Processor: RightSourceKeyed-SelectKey (stores: [RightSourceKeyed-Materialize]) + --> PrependAliasRight + <-- RightSourceKeyed-SelectKey-repartition-source + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000025 + <-- LeftSourceKeyed-SelectKey + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000026 + <-- RightSourceKeyed-SelectKey + Processor: KTABLE-JOINOTHER-0000000026 (stores: [LeftSourceKeyed-Materialize]) + --> KTABLE-MERGE-0000000024 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000025 (stores: [RightSourceKeyed-Materialize]) + --> KTABLE-MERGE-0000000024 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000024 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000025, KTABLE-JOINOTHER-0000000026 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000028 + <-- KTABLE-MERGE-0000000024 + Processor: KTABLE-TOSTREAM-0000000028 (stores: []) + --> KSTREAM-SINK-0000000029 + <-- Project + Sink: KSTREAM-SINK-0000000029 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000028 + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000013 (topics: [t2]) + --> KTABLE-SOURCE-0000000014 + Processor: KTABLE-SOURCE-0000000014 (stores: []) + --> KTABLE-MAPVALUES-0000000015 + <-- KSTREAM-SOURCE-0000000013 + Processor: KTABLE-MAPVALUES-0000000015 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000016 + <-- KTABLE-SOURCE-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000016 (stores: []) + --> KTABLE-TOSTREAM-0000000017 + <-- KTABLE-MAPVALUES-0000000015 + Processor: KTABLE-TOSTREAM-0000000017 (stores: []) + --> RightSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000016 + Processor: RightSourceKeyed-SelectKey-Mapper (stores: []) + --> RightSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000017 + Processor: RightSourceKeyed-SelectKey-repartition-filter (stores: []) + --> RightSourceKeyed-SelectKey-repartition-sink + <-- RightSourceKeyed-SelectKey-Mapper + Sink: RightSourceKeyed-SelectKey-repartition-sink (topic: RightSourceKeyed-SelectKey-repartition) + <-- RightSourceKeyed-SelectKey-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813451/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813451/plan.json new file mode 100644 index 000000000000..2e5607fe54ce --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813451/plan.json @@ -0,0 +1,243 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ROWKEY INTEGER PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='t1', KEY_FORMAT='JSON_SR', KEY_SCHEMA_ID=1, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ROWKEY` INTEGER KEY, `NAME` STRING", + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ROWKEY INTEGER PRIMARY KEY, F1 STRING) WITH (KAFKA_TOPIC='t2', KEY_FORMAT='JSON_SR', KEY_SCHEMA_ID=2, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ROWKEY` INTEGER KEY, `F1` STRING", + "topicName" : "t2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T1.ROWKEY T1_ROWKEY,\n T1.NAME NAME,\n T2.F1 F1\nFROM T1 T1\nLEFT OUTER JOIN T2 T2 ON ((T1.ROWKEY = T2.ROWKEY))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T1_ROWKEY` INTEGER KEY, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `NAME` STRING", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "T1_ROWKEY" ], + "selectExpressions" : [ "NAME AS T1_NAME", "ROWTIME AS T1_ROWTIME", "ROWKEY AS T1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `F1` STRING", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ROWKEY" + }, + "keyColumnNames" : [ "T2_ROWKEY" ], + "selectExpressions" : [ "F1 AS T2_F1", "ROWTIME AS T2_ROWTIME", "ROWKEY AS T2_ROWKEY" ] + }, + "keyColName" : "T1_ROWKEY" + }, + "keyColumnNames" : [ "T1_ROWKEY" ], + "selectExpressions" : [ "T1_NAME AS NAME", "T2_F1 AS F1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813451/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813451/spec.json new file mode 100644 index 000000000000..5e603aa4ace9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813451/spec.json @@ -0,0 +1,304 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799813451, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `NAME` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `F1` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T1_ROWKEY` INTEGER KEY, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.RightSourceKeyed.Materialize" : { + "schema" : "`ROWKEY` INTEGER KEY, `F1` STRING, `ROWTIME` BIGINT, `ROWKEY` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.LeftSourceKeyed.Materialize" : { + "schema" : "`ROWKEY` INTEGER KEY, `NAME` STRING, `ROWTIME` BIGINT, `ROWKEY` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "table-table - SR-enabled key format - with inference - different schemas", + "inputs" : [ { + "topic" : "t1", + "key" : 0, + "value" : { + "NAME" : "zero" + } + }, { + "topic" : "t2", + "key" : 0, + "value" : { + "F1" : "blah" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "F1" : null + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "F1" : "blah" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "t1", + "keySchema" : { + "type" : "integer", + "connect.type" : "int32" + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + }, { + "name" : "t2", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE TABLE T1 (NAME varchar) WITH (kafka_topic='t1', key_format='JSON_SR', value_format='JSON');", "CREATE TABLE T2 (F1 varchar) WITH (kafka_topic='t2', key_format='JSON_SR', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT T1.ROWKEY, name, f1 FROM T1 left join T2 on T1.ROWKEY = T2.ROWKEY;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`T1_ROWKEY` INTEGER KEY, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T1", + "type" : "TABLE", + "schema" : "`ROWKEY` INTEGER KEY, `NAME` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T2", + "type" : "TABLE", + "schema" : "`ROWKEY` INTEGER KEY, `F1` STRING", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-LeftSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-RightSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-LeftSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-RightSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "t2", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "t1", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 1, + "keySchema" : { + "type" : "integer", + "connect.type" : "int32" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813451/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813451/topology new file mode 100644 index 000000000000..0209cbd53ffd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_inference_-_different_schemas/6.2.0_1606799813451/topology @@ -0,0 +1,84 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [t1]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> KTABLE-TOSTREAM-0000000005 + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TOSTREAM-0000000005 (stores: []) + --> LeftSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: LeftSourceKeyed-SelectKey-Mapper (stores: []) + --> LeftSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000005 + Processor: LeftSourceKeyed-SelectKey-repartition-filter (stores: []) + --> LeftSourceKeyed-SelectKey-repartition-sink + <-- LeftSourceKeyed-SelectKey-Mapper + Sink: LeftSourceKeyed-SelectKey-repartition-sink (topic: LeftSourceKeyed-SelectKey-repartition) + <-- LeftSourceKeyed-SelectKey-repartition-filter + + Sub-topology: 1 + Source: LeftSourceKeyed-SelectKey-repartition-source (topics: [LeftSourceKeyed-SelectKey-repartition]) + --> LeftSourceKeyed-SelectKey + Source: RightSourceKeyed-SelectKey-repartition-source (topics: [RightSourceKeyed-SelectKey-repartition]) + --> RightSourceKeyed-SelectKey + Processor: LeftSourceKeyed-SelectKey (stores: [LeftSourceKeyed-Materialize]) + --> PrependAliasLeft + <-- LeftSourceKeyed-SelectKey-repartition-source + Processor: RightSourceKeyed-SelectKey (stores: [RightSourceKeyed-Materialize]) + --> PrependAliasRight + <-- RightSourceKeyed-SelectKey-repartition-source + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000025 + <-- LeftSourceKeyed-SelectKey + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000026 + <-- RightSourceKeyed-SelectKey + Processor: KTABLE-JOINOTHER-0000000026 (stores: [LeftSourceKeyed-Materialize]) + --> KTABLE-MERGE-0000000024 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000025 (stores: [RightSourceKeyed-Materialize]) + --> KTABLE-MERGE-0000000024 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000024 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000025, KTABLE-JOINOTHER-0000000026 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000028 + <-- KTABLE-MERGE-0000000024 + Processor: KTABLE-TOSTREAM-0000000028 (stores: []) + --> KSTREAM-SINK-0000000029 + <-- Project + Sink: KSTREAM-SINK-0000000029 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000028 + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000013 (topics: [t2]) + --> KTABLE-SOURCE-0000000014 + Processor: KTABLE-SOURCE-0000000014 (stores: []) + --> KTABLE-MAPVALUES-0000000015 + <-- KSTREAM-SOURCE-0000000013 + Processor: KTABLE-MAPVALUES-0000000015 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000016 + <-- KTABLE-SOURCE-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000016 (stores: []) + --> KTABLE-TOSTREAM-0000000017 + <-- KTABLE-MAPVALUES-0000000015 + Processor: KTABLE-TOSTREAM-0000000017 (stores: []) + --> RightSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000016 + Processor: RightSourceKeyed-SelectKey-Mapper (stores: []) + --> RightSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000017 + Processor: RightSourceKeyed-SelectKey-repartition-filter (stores: []) + --> RightSourceKeyed-SelectKey-repartition-sink + <-- RightSourceKeyed-SelectKey-Mapper + Sink: RightSourceKeyed-SelectKey-repartition-sink (topic: RightSourceKeyed-SelectKey-repartition) + <-- RightSourceKeyed-SelectKey-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_nulls/6.2.0_1606799812980/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_nulls/6.2.0_1606799812980/plan.json new file mode 100644 index 000000000000..c6a48bf38fb8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_nulls/6.2.0_1606799812980/plan.json @@ -0,0 +1,243 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, NAME STRING) WITH (KAFKA_TOPIC='t1', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, F1 STRING) WITH (KAFKA_TOPIC='t2', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` BIGINT KEY, `F1` STRING", + "topicName" : "t2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T1.ID T1_ID,\n T1.NAME NAME,\n T2.F1 F1\nFROM T1 T1\nLEFT OUTER JOIN T2 T2 ON ((T1.ID = T2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T1_ID` BIGINT KEY, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "NAME AS T1_NAME", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "F1 AS T2_F1", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T1_ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "T1_NAME AS NAME", "T2_F1 AS F1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_nulls/6.2.0_1606799812980/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_nulls/6.2.0_1606799812980/spec.json new file mode 100644 index 000000000000..c9c261f8c338 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_nulls/6.2.0_1606799812980/spec.json @@ -0,0 +1,273 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799812980, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T1_ID` BIGINT KEY, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.RightSourceKeyed.Materialize" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `ROWTIME` BIGINT, `ID` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.LeftSourceKeyed.Materialize" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `ROWTIME` BIGINT, `ID` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "table-table - SR-enabled key format - with nulls", + "inputs" : [ { + "topic" : "t1", + "key" : 0, + "value" : { + "NAME" : "zero" + } + }, { + "topic" : "t2", + "key" : 0, + "value" : { + "F1" : "blah" + } + }, { + "topic" : "t2", + "key" : 0, + "value" : null + }, { + "topic" : "t1", + "key" : 0, + "value" : null + }, { + "topic" : "t2", + "key" : 0, + "value" : { + "F1" : "foo" + } + }, { + "topic" : "t1", + "key" : 0, + "value" : { + "NAME" : "goo" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "F1" : null + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "F1" : "blah" + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "F1" : null + } + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "goo", + "F1" : "foo" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "t1", + "keySchema" : "long", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "t2", + "keySchema" : "long", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, NAME varchar) WITH (kafka_topic='t1', key_format='AVRO', value_format='JSON');", "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, F1 varchar) WITH (kafka_topic='t2', key_format='AVRO', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT T1.id, name, f1 FROM T1 left join T2 on T1.id = T2.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`T1_ID` BIGINT KEY, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T1", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-LeftSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-RightSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-LeftSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-RightSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "t2", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "long" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "long" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "long" + }, { + "name" : "t1", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "long" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_nulls/6.2.0_1606799812980/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_nulls/6.2.0_1606799812980/topology new file mode 100644 index 000000000000..0209cbd53ffd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table-table_-_SR-enabled_key_format_-_with_nulls/6.2.0_1606799812980/topology @@ -0,0 +1,84 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [t1]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> KTABLE-TOSTREAM-0000000005 + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TOSTREAM-0000000005 (stores: []) + --> LeftSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: LeftSourceKeyed-SelectKey-Mapper (stores: []) + --> LeftSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000005 + Processor: LeftSourceKeyed-SelectKey-repartition-filter (stores: []) + --> LeftSourceKeyed-SelectKey-repartition-sink + <-- LeftSourceKeyed-SelectKey-Mapper + Sink: LeftSourceKeyed-SelectKey-repartition-sink (topic: LeftSourceKeyed-SelectKey-repartition) + <-- LeftSourceKeyed-SelectKey-repartition-filter + + Sub-topology: 1 + Source: LeftSourceKeyed-SelectKey-repartition-source (topics: [LeftSourceKeyed-SelectKey-repartition]) + --> LeftSourceKeyed-SelectKey + Source: RightSourceKeyed-SelectKey-repartition-source (topics: [RightSourceKeyed-SelectKey-repartition]) + --> RightSourceKeyed-SelectKey + Processor: LeftSourceKeyed-SelectKey (stores: [LeftSourceKeyed-Materialize]) + --> PrependAliasLeft + <-- LeftSourceKeyed-SelectKey-repartition-source + Processor: RightSourceKeyed-SelectKey (stores: [RightSourceKeyed-Materialize]) + --> PrependAliasRight + <-- RightSourceKeyed-SelectKey-repartition-source + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000025 + <-- LeftSourceKeyed-SelectKey + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000026 + <-- RightSourceKeyed-SelectKey + Processor: KTABLE-JOINOTHER-0000000026 (stores: [LeftSourceKeyed-Materialize]) + --> KTABLE-MERGE-0000000024 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000025 (stores: [RightSourceKeyed-Materialize]) + --> KTABLE-MERGE-0000000024 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000024 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000025, KTABLE-JOINOTHER-0000000026 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000028 + <-- KTABLE-MERGE-0000000024 + Processor: KTABLE-TOSTREAM-0000000028 (stores: []) + --> KSTREAM-SINK-0000000029 + <-- Project + Sink: KSTREAM-SINK-0000000029 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000028 + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000013 (topics: [t2]) + --> KTABLE-SOURCE-0000000014 + Processor: KTABLE-SOURCE-0000000014 (stores: []) + --> KTABLE-MAPVALUES-0000000015 + <-- KSTREAM-SOURCE-0000000013 + Processor: KTABLE-MAPVALUES-0000000015 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000016 + <-- KTABLE-SOURCE-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000016 (stores: []) + --> KTABLE-TOSTREAM-0000000017 + <-- KTABLE-MAPVALUES-0000000015 + Processor: KTABLE-TOSTREAM-0000000017 (stores: []) + --> RightSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000016 + Processor: RightSourceKeyed-SelectKey-Mapper (stores: []) + --> RightSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000017 + Processor: RightSourceKeyed-SelectKey-repartition-filter (stores: []) + --> RightSourceKeyed-SelectKey-repartition-sink + <-- RightSourceKeyed-SelectKey-Mapper + Sink: RightSourceKeyed-SelectKey-repartition-sink (topic: RightSourceKeyed-SelectKey-repartition) + <-- RightSourceKeyed-SelectKey-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_inference/6.2.0_1606796984850/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_inference/6.2.0_1606796984850/plan.json new file mode 100644 index 000000000000..a87161c98580 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_inference/6.2.0_1606796984850/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY BIGINT KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_inference/6.2.0_1606796984850/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_inference/6.2.0_1606796984850/spec.json new file mode 100644 index 000000000000..200a4003781c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_inference/6.2.0_1606796984850/spec.json @@ -0,0 +1,180 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796984850, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "BIGINT - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 998877665544332211, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 998877665544332211, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_inference/6.2.0_1606796984850/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_inference/6.2.0_1606796984850/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_inference/6.2.0_1606796984850/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_no_inference/6.2.0_1606796984829/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_no_inference/6.2.0_1606796984829/plan.json new file mode 100644 index 000000000000..a88e907889cc --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_no_inference/6.2.0_1606796984829/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K BIGINT KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` BIGINT KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_no_inference/6.2.0_1606796984829/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_no_inference/6.2.0_1606796984829/spec.json new file mode 100644 index 000000000000..8c2fa3717f0c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_no_inference/6.2.0_1606796984829/spec.json @@ -0,0 +1,180 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796984829, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "BIGINT - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 998877665544332211, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 998877665544332211, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K BIGINT KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` BIGINT KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_no_inference/6.2.0_1606796984829/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_no_inference/6.2.0_1606796984829/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BIGINT_-_key_-_no_inference/6.2.0_1606796984829/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_inference/6.2.0_1606796984734/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_inference/6.2.0_1606796984734/plan.json new file mode 100644 index 000000000000..00449fa314c7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_inference/6.2.0_1606796984734/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY BOOLEAN KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_inference/6.2.0_1606796984734/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_inference/6.2.0_1606796984734/spec.json new file mode 100644 index 000000000000..e00ce873706e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_inference/6.2.0_1606796984734/spec.json @@ -0,0 +1,175 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796984734, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "BOOLEAN - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : true, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : null + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "boolean", + "connect.type" : "boolean" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "boolean", + "connect.type" : "boolean" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "boolean" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_inference/6.2.0_1606796984734/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_inference/6.2.0_1606796984734/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_inference/6.2.0_1606796984734/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796984688/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796984688/plan.json new file mode 100644 index 000000000000..9d347ae99a7a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796984688/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K BOOLEAN KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` BOOLEAN KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796984688/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796984688/spec.json new file mode 100644 index 000000000000..0495fd215b0d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796984688/spec.json @@ -0,0 +1,173 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796984688, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "BOOLEAN - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : true, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : null + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : true, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : null + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "boolean" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K BOOLEAN KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` BOOLEAN KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "boolean" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "boolean" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796984688/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796984688/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_BOOLEAN_-_key_-_no_inference/6.2.0_1606796984688/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_inference/6.2.0_1606796985191/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_inference/6.2.0_1606796985191/plan.json new file mode 100644 index 000000000000..cfd6d43ba947 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_inference/6.2.0_1606796985191/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY DECIMAL(4, 2) KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_inference/6.2.0_1606796985191/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_inference/6.2.0_1606796985191/spec.json new file mode 100644 index 000000000000..18905ca51c90 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_inference/6.2.0_1606796985191/spec.json @@ -0,0 +1,198 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796985191, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "DECIMAL - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 65.21, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 65.21, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "title" : "org.apache.kafka.connect.data.Decimal", + "connect.version" : 1, + "connect.type" : "bytes", + "connect.parameters" : { + "scale" : "2", + "connect.decimal.precision" : "4" + } + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "title" : "org.apache.kafka.connect.data.Decimal", + "connect.version" : 1, + "connect.type" : "bytes", + "connect.parameters" : { + "scale" : "2", + "connect.decimal.precision" : "4" + } + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "title" : "org.apache.kafka.connect.data.Decimal", + "connect.version" : 1, + "connect.type" : "bytes", + "connect.parameters" : { + "scale" : "2", + "connect.decimal.precision" : "4" + } + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_inference/6.2.0_1606796985191/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_inference/6.2.0_1606796985191/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_inference/6.2.0_1606796985191/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796985150/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796985150/plan.json new file mode 100644 index 000000000000..e352cca59684 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796985150/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K DECIMAL(4, 2) KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796985150/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796985150/spec.json new file mode 100644 index 000000000000..b7c20cab7227 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796985150/spec.json @@ -0,0 +1,198 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796985150, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "DECIMAL - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 65.21, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 65.21, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "title" : "org.apache.kafka.connect.data.Decimal", + "connect.version" : 1, + "connect.type" : "bytes", + "connect.parameters" : { + "scale" : "2", + "connect.decimal.precision" : "4" + } + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K DECIMAL(4,2) KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` DECIMAL(4, 2) KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "title" : "org.apache.kafka.connect.data.Decimal", + "connect.version" : 1, + "connect.type" : "bytes", + "connect.parameters" : { + "scale" : "2", + "connect.decimal.precision" : "4" + } + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "title" : "org.apache.kafka.connect.data.Decimal", + "connect.version" : 1, + "connect.type" : "bytes", + "connect.parameters" : { + "scale" : "2", + "connect.decimal.precision" : "4" + } + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796985150/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796985150/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DECIMAL_-_key_-_no_inference/6.2.0_1606796985150/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_inference/6.2.0_1606796984964/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_inference/6.2.0_1606796984964/plan.json new file mode 100644 index 000000000000..3be1e72fa0ed --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_inference/6.2.0_1606796984964/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY DOUBLE KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_inference/6.2.0_1606796984964/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_inference/6.2.0_1606796984964/spec.json new file mode 100644 index 000000000000..df552419887f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_inference/6.2.0_1606796984964/spec.json @@ -0,0 +1,180 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796984964, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "DOUBLE - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 654.321, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 654.321, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "connect.type" : "float64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "connect.type" : "float64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "connect.type" : "float64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_inference/6.2.0_1606796984964/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_inference/6.2.0_1606796984964/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_inference/6.2.0_1606796984964/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796984901/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796984901/plan.json new file mode 100644 index 000000000000..522dc847dd55 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796984901/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K DOUBLE KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` DOUBLE KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796984901/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796984901/spec.json new file mode 100644 index 000000000000..78d23c9b7892 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796984901/spec.json @@ -0,0 +1,180 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796984901, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "DOUBLE - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 654.321, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 654.321, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "connect.type" : "float64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K DOUBLE KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` DOUBLE KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "connect.type" : "float64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "number", + "connect.type" : "float64" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796984901/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796984901/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_DOUBLE_-_key_-_no_inference/6.2.0_1606796984901/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_inference/6.2.0_1606796984805/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_inference/6.2.0_1606796984805/plan.json new file mode 100644 index 000000000000..7f7bc273beca --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_inference/6.2.0_1606796984805/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY INTEGER KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_inference/6.2.0_1606796984805/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_inference/6.2.0_1606796984805/spec.json new file mode 100644 index 000000000000..3475f2623d94 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_inference/6.2.0_1606796984805/spec.json @@ -0,0 +1,192 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796984805, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "INT - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_inference/6.2.0_1606796984805/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_inference/6.2.0_1606796984805/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_inference/6.2.0_1606796984805/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796985258/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796985258/plan.json new file mode 100644 index 000000000000..5783ff3ce1fa --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796985258/plan.json @@ -0,0 +1,149 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY INTEGER KEY, ROWVAL INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1, VALUE_SCHEMA_ID=1, WRAP_SINGLE_VALUE=false);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "ROWVAL AS ROWVAL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796985258/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796985258/spec.json new file mode 100644 index 000000000000..32df65ef8db8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796985258/spec.json @@ -0,0 +1,174 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796985258, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "INT - key - key and anonymous value inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 10, + "value" : 10 + }, { + "topic" : "input_topic", + "key" : 12, + "value" : null + }, { + "topic" : "input_topic", + "key" : null, + "value" : 10 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "ROWVAL" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : 12, + "value" : null + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "ROWVAL" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT WITH (kafka_topic='input_topic', format='JSON_SR', wrap_single_value=false);", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ "UNWRAP_SINGLES" ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `ROWVAL` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "ROWVAL" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796985258/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796985258/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_anonymous_value_inference/6.2.0_1606796985258/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796985227/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796985227/plan.json new file mode 100644 index 000000000000..21554bf9db59 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796985227/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY INTEGER KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1, VALUE_SCHEMA_ID=2);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796985227/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796985227/spec.json new file mode 100644 index 000000000000..2a4871fed3de --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796985227/spec.json @@ -0,0 +1,192 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796985227, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "INT - key - key and value inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796985227/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796985227/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_key_and_value_inference/6.2.0_1606796985227/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_no_inference/6.2.0_1606796984776/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_no_inference/6.2.0_1606796984776/plan.json new file mode 100644 index 000000000000..c6cf247c7578 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_no_inference/6.2.0_1606796984776/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K INTEGER KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` INTEGER KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_no_inference/6.2.0_1606796984776/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_no_inference/6.2.0_1606796984776/spec.json new file mode 100644 index 000000000000..462df7ca0ac6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_no_inference/6.2.0_1606796984776/spec.json @@ -0,0 +1,192 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796984776, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "INT - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : 12, + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K INT KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` INTEGER KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_no_inference/6.2.0_1606796984776/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_no_inference/6.2.0_1606796984776/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_INT_-_key_-_no_inference/6.2.0_1606796984776/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_inference/6.2.0_1606796985074/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_inference/6.2.0_1606796985074/plan.json new file mode 100644 index 000000000000..d5dd46e9852c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_inference/6.2.0_1606796985074/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ROWKEY STRING KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic', KEY_SCHEMA_ID=1);", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ROWKEY` STRING KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "ROWKEY" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_inference/6.2.0_1606796985074/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_inference/6.2.0_1606796985074/spec.json new file mode 100644 index 000000000000..fade13e31bb6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_inference/6.2.0_1606796985074/spec.json @@ -0,0 +1,179 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796985074, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "STRING - key - inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : "foo", + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "foo", + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "string", + "connect.type" : "string" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ROWKEY` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 1, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "string", + "connect.type" : "string" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "COL1" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "string" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_inference/6.2.0_1606796985074/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_inference/6.2.0_1606796985074/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_inference/6.2.0_1606796985074/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_no_inference/6.2.0_1606796985012/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_no_inference/6.2.0_1606796985012/plan.json new file mode 100644 index 000000000000..34e1e2a91bf5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_no_inference/6.2.0_1606796985012/plan.json @@ -0,0 +1,147 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (K STRING KEY, FOO INTEGER) WITH (FORMAT='JSON_SR', KAFKA_TOPIC='input_topic');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "input_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`K` STRING KEY, `FOO` INTEGER" + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "FOO AS FOO" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_no_inference/6.2.0_1606796985012/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_no_inference/6.2.0_1606796985012/spec.json new file mode 100644 index 000000000000..7814c2aa285c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_no_inference/6.2.0_1606796985012/spec.json @@ -0,0 +1,177 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606796985012, + "path" : "query-validation-tests/json_sr.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + } + } + }, + "testCase" : { + "name" : "STRING - key - no inference", + "inputs" : [ { + "topic" : "input_topic", + "key" : "foo", + "value" : { + "FOO" : 10 + } + }, { + "topic" : "input_topic", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "foo", + "value" : { + "FOO" : 10 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "FOO" : 10 + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "input_topic", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "string" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + }, + "keyFormat" : "JSON", + "valueFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (K VARCHAR KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `FOO` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON_SR", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "input_topic", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "string" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON_SR" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "string" + } ] + }, + "valueSchema" : { + "type" : "object", + "properties" : { + "FOO" : { + "connect.index" : 0, + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_no_inference/6.2.0_1606796985012/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_no_inference/6.2.0_1606796985012/topology new file mode 100644 index 000000000000..12f8f6574002 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/json_sr_-_STRING_-_key_-_no_inference/6.2.0_1606796985012/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [input_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format/6.2.0_1606799824556/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format/6.2.0_1606799824556/plan.json new file mode 100644 index 000000000000..3c2cb3a55257 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format/6.2.0_1606799824556/plan.json @@ -0,0 +1,322 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', KEY_FORMAT='JSON_SR', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S3 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S2.ID S2_ID,\n S1.V0 S1_V0,\n S2.V0 S2_V0,\n S3.V0 S3_V0\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 5 SECONDS ON ((S1.ID = S2.ID))\nINNER JOIN S3 S3 WITHIN 10 SECONDS ON ((S1.ID = S3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `S3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2", "S3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "V0 AS S2_V0", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 5.000000000, + "afterMillis" : 5.000000000, + "keyColName" : "S2_ID" + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S3_ID" ], + "selectExpressions" : [ "V0 AS S3_V0", "ROWTIME AS S3_ROWTIME", "ID AS S3_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S2_ID" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "S2_V0 AS S2_V0", "S3_V0 AS S3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format/6.2.0_1606799824556/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format/6.2.0_1606799824556/spec.json new file mode 100644 index 000000000000..e93dcbd69f91 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format/6.2.0_1606799824556/spec.json @@ -0,0 +1,481 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799824556, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.L_Join.Left" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.L_Join.Right" : { + "schema" : "`S2_ID` INTEGER KEY, `S2_V0` BIGINT, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S2_V0` BIGINT, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.L_Join" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `S3_V0` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S2_V0` BIGINT, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`S3_ID` INTEGER KEY, `S3_V0` BIGINT, `S3_ROWTIME` BIGINT, `S3_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "stream-stream-stream - SR-enabled key format", + "inputs" : [ { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 0 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 1 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 2 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 3 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 5 + }, + "timestamp" : 4000 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 6 + }, + "timestamp" : 6000 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 7 + }, + "timestamp" : 6001 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 8 + }, + "timestamp" : 7000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 2, + "S3_V0" : 3 + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 4, + "S2_V0" : 2, + "S3_V0" : 3 + }, + "timestamp" : 3 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : 2, + "S3_V0" : 3 + }, + "timestamp" : 4000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 2, + "S3_V0" : 6 + }, + "timestamp" : 6000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 4, + "S2_V0" : 2, + "S3_V0" : 6 + }, + "timestamp" : 6000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : 2, + "S3_V0" : 6 + }, + "timestamp" : 6000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 2, + "S3_V0" : 8 + }, + "timestamp" : 7000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 4, + "S2_V0" : 2, + "S3_V0" : 8 + }, + "timestamp" : 7000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : 2, + "S3_V0" : 8 + }, + "timestamp" : 7000 + } ], + "topics" : [ { + "name" : "left", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, V0 bigint) WITH (kafka_topic='right', key_format='AVRO', value_format='JSON');", "CREATE STREAM S3 (ID INT KEY, V0 bigint) WITH (kafka_topic='right2', key_format='AVRO', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S2.ID, s1.V0, s2.V0, s3.V0 FROM S1 JOIN S2 WITHIN 5 seconds ON S1.ID = S2.ID JOIN S3 WITHIN 10 seconds ON S1.ID = S3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `S3_V0` BIGINT", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S3", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-left-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000029-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-right-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "left", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000028-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "right2", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "right", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format/6.2.0_1606799824556/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format/6.2.0_1606799824556/topology new file mode 100644 index 000000000000..fc5c6465a2e5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format/6.2.0_1606799824556/topology @@ -0,0 +1,98 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> L_LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: L_LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-left-repartition-filter + <-- L_LeftSourceKeyed-SelectKey + Processor: L_Join-left-repartition-filter (stores: []) + --> L_Join-left-repartition-sink + <-- PrependAliasL_Left + Sink: L_Join-left-repartition-sink (topic: L_Join-left-repartition) + <-- L_Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> L_RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: L_RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Right + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasL_Right (stores: []) + --> L_Join-right-repartition-filter + <-- L_RightSourceKeyed-SelectKey + Processor: L_Join-right-repartition-filter (stores: []) + --> L_Join-right-repartition-sink + <-- PrependAliasL_Right + Sink: L_Join-right-repartition-sink (topic: L_Join-right-repartition) + <-- L_Join-right-repartition-filter + + Sub-topology: 2 + Source: L_Join-left-repartition-source (topics: [L_Join-left-repartition]) + --> L_Join-this-windowed + Source: L_Join-right-repartition-source (topics: [L_Join-right-repartition]) + --> L_Join-other-windowed + Processor: L_Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> L_Join-other-join + <-- L_Join-right-repartition-source + Processor: L_Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> L_Join-this-join + <-- L_Join-left-repartition-source + Processor: L_Join-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> L_Join-merge + <-- L_Join-other-windowed + Processor: L_Join-this-join (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> L_Join-merge + <-- L_Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: L_Join-merge (stores: []) + --> Join-this-windowed + <-- L_Join-this-join, L_Join-other-join + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000029-store]) + --> Join-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000028-store]) + --> Join-this-join + <-- L_Join-merge + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000028-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000029-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000032 + <-- Join-merge + Sink: KSTREAM-SINK-0000000032 (topic: OUTPUT) + <-- Project + + Sub-topology: 3 + Source: KSTREAM-SOURCE-0000000019 (topics: [right2]) + --> KSTREAM-TRANSFORMVALUES-0000000020 + Processor: KSTREAM-TRANSFORMVALUES-0000000020 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000019 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000020 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format_-_rekey/6.2.0_1606799824701/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format_-_rekey/6.2.0_1606799824701/plan.json new file mode 100644 index 000000000000..f02b19136d73 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format_-_rekey/6.2.0_1606799824701/plan.json @@ -0,0 +1,329 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 INTEGER) WITH (KAFKA_TOPIC='left', KEY_FORMAT='JSON_SR', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, V0 INTEGER) WITH (KAFKA_TOPIC='right', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S3 (ID INTEGER KEY, V0 INTEGER) WITH (KAFKA_TOPIC='right2', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S3", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S2.ID S2_ID,\n S1.V0 S1_V0,\n S2.V0 S2_V0,\n S3.V0 S3_V0\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 5 SECONDS ON ((S1.ID = S2.ID))\nINNER JOIN S3 S3 WITHIN 10 SECONDS ON ((S1.V0 = S3.V0))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_V0` INTEGER KEY, `S2_ID` INTEGER, `S2_V0` INTEGER, `S3_V0` INTEGER", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2", "S3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "V0 AS S2_V0", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 5.000000000, + "afterMillis" : 5.000000000, + "keyColName" : "S2_ID" + }, + "keyExpression" : "S1_V0" + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER" + }, + "keyExpression" : "V0" + }, + "keyColumnNames" : [ "S3_V0" ], + "selectExpressions" : [ "V0 AS S3_V0", "ROWTIME AS S3_ROWTIME", "ID AS S3_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S1_V0" + }, + "keyColumnNames" : [ "S1_V0" ], + "selectExpressions" : [ "S2_ID AS S2_ID", "S2_V0 AS S2_V0", "S3_V0 AS S3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format_-_rekey/6.2.0_1606799824701/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format_-_rekey/6.2.0_1606799824701/spec.json new file mode 100644 index 000000000000..c21bfcec61c6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format_-_rekey/6.2.0_1606799824701/spec.json @@ -0,0 +1,391 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799824701, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.L_Join.Left" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` INTEGER, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.L_Join.Right" : { + "schema" : "`S2_ID` INTEGER KEY, `S2_V0` INTEGER, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S1_V0` INTEGER KEY, `S1_V0` INTEGER, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S2_V0` INTEGER, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.L_Join" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` INTEGER, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S1_V0` INTEGER KEY, `S2_ID` INTEGER, `S2_V0` INTEGER, `S3_V0` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S1_V0` INTEGER KEY, `S1_V0` INTEGER, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S2_V0` INTEGER, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`S3_V0` INTEGER KEY, `S3_V0` INTEGER, `S3_ROWTIME` BIGINT, `S3_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "stream-stream-stream - SR-enabled key format - rekey", + "inputs" : [ { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 0 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 1 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 2 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 3, + "value" : { + "S2_ID" : 0, + "S2_V0" : 2, + "S3_V0" : 3 + }, + "timestamp" : 2 + } ], + "topics" : [ { + "name" : "left", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 INT) WITH (kafka_topic='left', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, V0 INT) WITH (kafka_topic='right', key_format='AVRO', value_format='JSON');", "CREATE STREAM S3 (ID INT KEY, V0 INT) WITH (kafka_topic='right2', key_format='AVRO', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S2.ID, s1.V0, s2.V0, s3.V0 FROM S1 JOIN S2 WITHIN 5 seconds ON S1.ID = S2.ID JOIN S3 WITHIN 10 seconds ON S1.V0 = S3.V0;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S1_V0` INTEGER KEY, `S2_ID` INTEGER, `S2_V0` INTEGER, `S3_V0` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S3", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-left-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "left", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-right-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000032-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "right2", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000033-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "right", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format_-_rekey/6.2.0_1606799824701/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format_-_rekey/6.2.0_1606799824701/topology new file mode 100644 index 000000000000..2e94f00b0e2d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-stream_-_SR-enabled_key_format_-_rekey/6.2.0_1606799824701/topology @@ -0,0 +1,110 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> L_LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: L_LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-left-repartition-filter + <-- L_LeftSourceKeyed-SelectKey + Processor: L_Join-left-repartition-filter (stores: []) + --> L_Join-left-repartition-sink + <-- PrependAliasL_Left + Sink: L_Join-left-repartition-sink (topic: L_Join-left-repartition) + <-- L_Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> L_RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: L_RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Right + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasL_Right (stores: []) + --> L_Join-right-repartition-filter + <-- L_RightSourceKeyed-SelectKey + Processor: L_Join-right-repartition-filter (stores: []) + --> L_Join-right-repartition-sink + <-- PrependAliasL_Right + Sink: L_Join-right-repartition-sink (topic: L_Join-right-repartition) + <-- L_Join-right-repartition-filter + + Sub-topology: 2 + Source: L_Join-left-repartition-source (topics: [L_Join-left-repartition]) + --> L_Join-this-windowed + Source: L_Join-right-repartition-source (topics: [L_Join-right-repartition]) + --> L_Join-other-windowed + Processor: L_Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> L_Join-other-join + <-- L_Join-right-repartition-source + Processor: L_Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> L_Join-this-join + <-- L_Join-left-repartition-source + Processor: L_Join-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> L_Join-merge + <-- L_Join-other-windowed + Processor: L_Join-this-join (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> L_Join-merge + <-- L_Join-this-windowed + Processor: L_Join-merge (stores: []) + --> LeftSourceKeyed-SelectKey + <-- L_Join-this-join, L_Join-other-join + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> Join-left-repartition-filter + <-- L_Join-merge + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- LeftSourceKeyed-SelectKey + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 3 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000033-store]) + --> Join-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000032-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000032-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000033-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000036 + <-- Join-merge + Sink: KSTREAM-SINK-0000000036 (topic: OUTPUT) + <-- Project + + Sub-topology: 4 + Source: KSTREAM-SOURCE-0000000020 (topics: [right2]) + --> KSTREAM-TRANSFORMVALUES-0000000021 + Processor: KSTREAM-TRANSFORMVALUES-0000000021 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000020 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000021 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format/6.2.0_1606799824831/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format/6.2.0_1606799824831/plan.json new file mode 100644 index 000000000000..b310223bfb9c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format/6.2.0_1606799824831/plan.json @@ -0,0 +1,321 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', KEY_FORMAT='JSON_SR', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', KEY_FORMAT='JSON_SR', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S2.ID S2_ID,\n S1.V0 S1_V0,\n S2.V0 S2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.ID = S2.ID))\nINNER JOIN T3 T3 ON ((S1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "V0 AS S2_V0", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S2_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S2_ID" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "S1_V0 AS S1_V0", "S2_V0 AS S2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format/6.2.0_1606799824831/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format/6.2.0_1606799824831/spec.json new file mode 100644 index 000000000000..e553a27609a0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format/6.2.0_1606799824831/spec.json @@ -0,0 +1,424 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799824831, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.L_Join.Left" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.L_Join.Right" : { + "schema" : "`S2_ID` INTEGER KEY, `S2_V0` BIGINT, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S2_V0` BIGINT, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.L_Join" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `T3_V0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.RightSourceKeyed.Materialize" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT, `ROWTIME` BIGINT, `ID` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S2_V0` BIGINT, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "stream-stream-table - SR-enabled key format", + "inputs" : [ { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 13 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 5 + }, + "timestamp" : 100000 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 6 + }, + "timestamp" : 100001 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 1, + "S2_V0" : 4, + "T3_V0" : 3 + }, + "timestamp" : 13 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "S1_V0" : 5, + "S2_V0" : 6, + "T3_V0" : 3 + }, + "timestamp" : 100001 + } ], + "topics" : [ { + "name" : "left", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 bigint) WITH (kafka_topic='left', key_format='AVRO', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, V0 bigint) WITH (kafka_topic='right', key_format='JSON_SR', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S2.ID, s1.V0, s2.V0, t3.V0 FROM S1 JOIN S2 WITHIN 10 seconds ON S1.ID = S2.ID JOIN T3 ON S1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S2_ID` INTEGER KEY, `S1_V0` BIGINT, `S2_V0` BIGINT, `T3_V0` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T3", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-left-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000029-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "left", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-right-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000028-store-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-RightSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "right2", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "right", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-RightSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format/6.2.0_1606799824831/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format/6.2.0_1606799824831/topology new file mode 100644 index 000000000000..e1d36cb11f6a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format/6.2.0_1606799824831/topology @@ -0,0 +1,98 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> KTABLE-TOSTREAM-0000000005 + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TOSTREAM-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: RightSourceKeyed-SelectKey-Mapper (stores: []) + --> RightSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000005 + Processor: RightSourceKeyed-SelectKey-repartition-filter (stores: []) + --> RightSourceKeyed-SelectKey-repartition-sink + <-- RightSourceKeyed-SelectKey-Mapper + Sink: RightSourceKeyed-SelectKey-repartition-sink (topic: RightSourceKeyed-SelectKey-repartition) + <-- RightSourceKeyed-SelectKey-repartition-filter + + Sub-topology: 1 + Source: L_Join-left-repartition-source (topics: [L_Join-left-repartition]) + --> L_Join-this-windowed + Source: L_Join-right-repartition-source (topics: [L_Join-right-repartition]) + --> L_Join-other-windowed + Processor: L_Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000029-store]) + --> L_Join-other-join + <-- L_Join-right-repartition-source + Processor: L_Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000028-store]) + --> L_Join-this-join + <-- L_Join-left-repartition-source + Processor: L_Join-other-join (stores: [KSTREAM-JOINTHIS-0000000028-store]) + --> L_Join-merge + <-- L_Join-other-windowed + Processor: L_Join-this-join (stores: [KSTREAM-JOINOTHER-0000000029-store]) + --> L_Join-merge + <-- L_Join-this-windowed + Processor: L_Join-merge (stores: []) + --> Join + <-- L_Join-this-join, L_Join-other-join + Processor: Join (stores: [RightSourceKeyed-Materialize]) + --> Project + <-- L_Join-merge + Source: RightSourceKeyed-SelectKey-repartition-source (topics: [RightSourceKeyed-SelectKey-repartition]) + --> RightSourceKeyed-SelectKey + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000033 + <-- Join + Processor: RightSourceKeyed-SelectKey (stores: [RightSourceKeyed-Materialize]) + --> PrependAliasRight + <-- RightSourceKeyed-SelectKey-repartition-source + Sink: KSTREAM-SINK-0000000033 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- RightSourceKeyed-SelectKey + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> L_LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000012 + Processor: L_LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-left-repartition-filter + <-- L_LeftSourceKeyed-SelectKey + Processor: L_Join-left-repartition-filter (stores: []) + --> L_Join-left-repartition-sink + <-- PrependAliasL_Left + Sink: L_Join-left-repartition-sink (topic: L_Join-left-repartition) + <-- L_Join-left-repartition-filter + + Sub-topology: 3 + Source: KSTREAM-SOURCE-0000000016 (topics: [right]) + --> KSTREAM-TRANSFORMVALUES-0000000017 + Processor: KSTREAM-TRANSFORMVALUES-0000000017 (stores: []) + --> L_RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000016 + Processor: L_RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Right + <-- KSTREAM-TRANSFORMVALUES-0000000017 + Processor: PrependAliasL_Right (stores: []) + --> L_Join-right-repartition-filter + <-- L_RightSourceKeyed-SelectKey + Processor: L_Join-right-repartition-filter (stores: []) + --> L_Join-right-repartition-sink + <-- PrependAliasL_Right + Sink: L_Join-right-repartition-sink (topic: L_Join-right-repartition) + <-- L_Join-right-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format_-_rekey_-_differing_internal_key_formats/6.2.0_1606799825011/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format_-_rekey_-_differing_internal_key_formats/6.2.0_1606799825011/plan.json new file mode 100644 index 000000000000..87852dfb8def --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format_-_rekey_-_differing_internal_key_formats/6.2.0_1606799825011/plan.json @@ -0,0 +1,328 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID INTEGER KEY, V0 INTEGER) WITH (KAFKA_TOPIC='left', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID INTEGER KEY, V0 INTEGER) WITH (KAFKA_TOPIC='right', KEY_FORMAT='JSON_SR', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', KEY_FORMAT='JSON_SR', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n S2.ID S2_ID,\n S1.V0 S1_V0,\n S2.V0 S2_V0,\n T3.V0 T3_V0\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 10 SECONDS ON ((S1.ID = S2.ID))\nINNER JOIN T3 T3 ON ((S1.V0 = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`S1_V0` INTEGER KEY, `S2_ID` INTEGER, `S2_V0` INTEGER, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "V0 AS S1_V0", "ROWTIME AS S1_ROWTIME", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "L_RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` INTEGER" + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "V0 AS S2_V0", "ROWTIME AS S2_ROWTIME", "ID AS S2_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "keyColName" : "S2_ID" + }, + "keyExpression" : "S1_V0" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "S1_V0" + }, + "keyColumnNames" : [ "S1_V0" ], + "selectExpressions" : [ "S2_ID AS S2_ID", "S2_V0 AS S2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format_-_rekey_-_differing_internal_key_formats/6.2.0_1606799825011/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format_-_rekey_-_differing_internal_key_formats/6.2.0_1606799825011/spec.json new file mode 100644 index 000000000000..9201a20c337d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format_-_rekey_-_differing_internal_key_formats/6.2.0_1606799825011/spec.json @@ -0,0 +1,374 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799825011, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CSAS_OUTPUT_0.L_Join.Left" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` INTEGER, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_L_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.L_Join.Right" : { + "schema" : "`S2_ID` INTEGER KEY, `S2_V0` INTEGER, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`S1_V0` INTEGER KEY, `S1_V0` INTEGER, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S2_V0` INTEGER, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.L_Join" : { + "schema" : "`S1_ID` INTEGER KEY, `S1_V0` INTEGER, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`S1_V0` INTEGER KEY, `S2_ID` INTEGER, `S2_V0` INTEGER, `T3_V0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.RightSourceKeyed.Materialize" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT, `ROWTIME` BIGINT, `ID` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`S1_V0` INTEGER KEY, `S1_V0` INTEGER, `S1_ROWTIME` BIGINT, `S1_ID` INTEGER, `S2_V0` INTEGER, `S2_ROWTIME` BIGINT, `S2_ID` INTEGER", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + } + }, + "CSAS_OUTPUT_0.KafkaTopic_L_Left.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "stream-stream-table - SR-enabled key format - rekey - differing internal key formats", + "inputs" : [ { + "topic" : "right2", + "key" : 1, + "value" : { + "V0" : 3 + }, + "timestamp" : 10 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 11 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 12 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "S2_ID" : 0, + "S2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 12 + } ], + "topics" : [ { + "name" : "left", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + }, + "keyFormat" : "JSON", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID INT KEY, V0 INT) WITH (kafka_topic='left', key_format='AVRO', value_format='JSON');", "CREATE STREAM S2 (ID INT KEY, V0 INT) WITH (kafka_topic='right', key_format='JSON_SR', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S2.ID, s1.V0, s2.V0, t3.V0 FROM S1 JOIN S2 WITHIN 10 seconds ON S1.ID = S2.ID JOIN T3 ON S1.V0 = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`S1_V0` INTEGER KEY, `S2_ID` INTEGER, `S2_V0` INTEGER, `T3_V0` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ID` INTEGER KEY, `V0` INTEGER", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T3", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "JSON_SR" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-left-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000029-store-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "left", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-L_Join-right-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000028-store-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-RightSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "right2", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-repartition", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "right", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-RightSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "JSON_SR", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : { + "oneOf" : [ { + "type" : "null" + }, { + "type" : "integer", + "connect.type" : "int32" + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format_-_rekey_-_differing_internal_key_formats/6.2.0_1606799825011/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format_-_rekey_-_differing_internal_key_formats/6.2.0_1606799825011/topology new file mode 100644 index 000000000000..886e81ad0699 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_stream-stream-table_-_SR-enabled_key_format_-_rekey_-_differing_internal_key_formats/6.2.0_1606799825011/topology @@ -0,0 +1,110 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [right2]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> KTABLE-TOSTREAM-0000000005 + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TOSTREAM-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: RightSourceKeyed-SelectKey-Mapper (stores: []) + --> RightSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000005 + Processor: RightSourceKeyed-SelectKey-repartition-filter (stores: []) + --> RightSourceKeyed-SelectKey-repartition-sink + <-- RightSourceKeyed-SelectKey-Mapper + Sink: RightSourceKeyed-SelectKey-repartition-sink (topic: RightSourceKeyed-SelectKey-repartition) + <-- RightSourceKeyed-SelectKey-repartition-filter + + Sub-topology: 1 + Source: Join-repartition-source (topics: [Join-repartition]) + --> Join + Processor: Join (stores: [RightSourceKeyed-Materialize]) + --> Project + <-- Join-repartition-source + Source: RightSourceKeyed-SelectKey-repartition-source (topics: [RightSourceKeyed-SelectKey-repartition]) + --> RightSourceKeyed-SelectKey + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000037 + <-- Join + Processor: RightSourceKeyed-SelectKey (stores: [RightSourceKeyed-Materialize]) + --> PrependAliasRight + <-- RightSourceKeyed-SelectKey-repartition-source + Sink: KSTREAM-SINK-0000000037 (topic: OUTPUT) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- RightSourceKeyed-SelectKey + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000012 (topics: [left]) + --> KSTREAM-TRANSFORMVALUES-0000000013 + Processor: KSTREAM-TRANSFORMVALUES-0000000013 (stores: []) + --> L_LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000012 + Processor: L_LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Left + <-- KSTREAM-TRANSFORMVALUES-0000000013 + Processor: PrependAliasL_Left (stores: []) + --> L_Join-left-repartition-filter + <-- L_LeftSourceKeyed-SelectKey + Processor: L_Join-left-repartition-filter (stores: []) + --> L_Join-left-repartition-sink + <-- PrependAliasL_Left + Sink: L_Join-left-repartition-sink (topic: L_Join-left-repartition) + <-- L_Join-left-repartition-filter + + Sub-topology: 3 + Source: KSTREAM-SOURCE-0000000016 (topics: [right]) + --> KSTREAM-TRANSFORMVALUES-0000000017 + Processor: KSTREAM-TRANSFORMVALUES-0000000017 (stores: []) + --> L_RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000016 + Processor: L_RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasL_Right + <-- KSTREAM-TRANSFORMVALUES-0000000017 + Processor: PrependAliasL_Right (stores: []) + --> L_Join-right-repartition-filter + <-- L_RightSourceKeyed-SelectKey + Processor: L_Join-right-repartition-filter (stores: []) + --> L_Join-right-repartition-sink + <-- PrependAliasL_Right + Sink: L_Join-right-repartition-sink (topic: L_Join-right-repartition) + <-- L_Join-right-repartition-filter + + Sub-topology: 4 + Source: L_Join-left-repartition-source (topics: [L_Join-left-repartition]) + --> L_Join-this-windowed + Source: L_Join-right-repartition-source (topics: [L_Join-right-repartition]) + --> L_Join-other-windowed + Processor: L_Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000029-store]) + --> L_Join-other-join + <-- L_Join-right-repartition-source + Processor: L_Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000028-store]) + --> L_Join-this-join + <-- L_Join-left-repartition-source + Processor: L_Join-other-join (stores: [KSTREAM-JOINTHIS-0000000028-store]) + --> L_Join-merge + <-- L_Join-other-windowed + Processor: L_Join-this-join (stores: [KSTREAM-JOINOTHER-0000000029-store]) + --> L_Join-merge + <-- L_Join-this-windowed + Processor: L_Join-merge (stores: []) + --> LeftSourceKeyed-SelectKey + <-- L_Join-this-join, L_Join-other-join + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> Join-repartition-filter + <-- L_Join-merge + Processor: Join-repartition-filter (stores: []) + --> Join-repartition-sink + <-- LeftSourceKeyed-SelectKey + Sink: Join-repartition-sink (topic: Join-repartition) + <-- Join-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_SR-enabled_key_format/6.2.0_1606799825157/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_SR-enabled_key_format/6.2.0_1606799825157/plan.json new file mode 100644 index 000000000000..83429e7ec1e1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_SR-enabled_key_format/6.2.0_1606799825157/plan.json @@ -0,0 +1,312 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='left', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T2 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T2", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T3 (ID INTEGER PRIMARY KEY, V0 BIGINT) WITH (KAFKA_TOPIC='right2', KEY_FORMAT='AVRO', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T3", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n ROWKEY ID,\n T1.V0 T1_V0,\n T2.V0 T2_V0,\n T3.V0 T3_V0\nFROM T1 T1\nINNER JOIN T2 T2 ON ((T1.ID = T2.ID))\nFULL OUTER JOIN T3 T3 ON ((T1.ID = T3.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ID` INTEGER KEY, `T1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "T1", "T2", "T3" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftSource" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "L_LeftSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "V0 AS T1_V0", "ROWTIME AS T1_ROWTIME", "ID AS T1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "L_RightSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "right", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "T2_ID" ], + "selectExpressions" : [ "V0 AS T2_V0", "ROWTIME AS T2_ROWTIME", "ID AS T2_ID" ] + }, + "keyColName" : "T1_ID" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSelectKeyV1", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right2", + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID` INTEGER KEY, `V0` BIGINT", + "forceChangelog" : true + }, + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "keyExpression" : "ID" + }, + "keyColumnNames" : [ "T3_ID" ], + "selectExpressions" : [ "V0 AS T3_V0", "ROWTIME AS T3_ROWTIME", "ID AS T3_ID" ] + }, + "keyColName" : "ROWKEY" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "T1_V0 AS T1_V0", "T2_V0 AS T2_V0", "T3_V0 AS T3_V0" ] + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.key.format.enabled" : "false", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "false", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.multicol.key.format.enabled" : "false", + "ksql.suppress.enabled" : "false", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.complex.key.format.enabled" : "false", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_SR-enabled_key_format/6.2.0_1606799825157/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_SR-enabled_key_format/6.2.0_1606799825157/spec.json new file mode 100644 index 000000000000..665729139f99 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_SR-enabled_key_format/6.2.0_1606799825157/spec.json @@ -0,0 +1,341 @@ +{ + "version" : "6.2.0", + "timestamp" : 1606799825157, + "path" : "query-validation-tests/multi-joins.json", + "schemas" : { + "CTAS_OUTPUT_0.L_LeftSourceKeyed.Materialize" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT, `ROWTIME` BIGINT, `ID` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Left.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ID` INTEGER KEY, `T1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.RightSourceKeyed.Materialize" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT, `ROWTIME` BIGINT, `ID` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.L_RightSourceKeyed.Materialize" : { + "schema" : "`ID` INTEGER KEY, `V0` BIGINT, `ROWTIME` BIGINT, `ID` INTEGER", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "table-table-table - SR-enabled key format", + "inputs" : [ { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 1 + }, + "timestamp" : 0 + }, { + "topic" : "right2", + "key" : 0, + "value" : { + "V0" : 3 + }, + "timestamp" : 2 + }, { + "topic" : "right", + "key" : 0, + "value" : { + "V0" : 2 + }, + "timestamp" : 3 + }, { + "topic" : "left", + "key" : 0, + "value" : { + "V0" : 4 + }, + "timestamp" : 1000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T1_V0" : null, + "T2_V0" : null, + "T3_V0" : 3 + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T1_V0" : 1, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 3 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T1_V0" : 4, + "T2_V0" : 2, + "T3_V0" : 3 + }, + "timestamp" : 1000 + } ], + "topics" : [ { + "name" : "left", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right2", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right", + "keySchema" : "int", + "keyFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE T1 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='left', key_format='AVRO', value_format='JSON');", "CREATE TABLE T2 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right', key_format='AVRO', value_format='JSON');", "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', key_format='AVRO', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT ROWKEY AS ID, T1.V0, T2.V0, T3.V0 FROM T1 JOIN T2 ON T1.ID = T2.ID FULL JOIN T3 ON T1.ID = T3.ID;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `T1_V0` BIGINT, `T2_V0` BIGINT, `T3_V0` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T1", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T2", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "T3", + "type" : "TABLE", + "schema" : "`ID` INTEGER KEY, `V0` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-L_RightSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-L_LeftSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-L_RightSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-L_LeftSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-RightSourceKeyed-Materialize-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "left", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Left-Reduce-changelog", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-RightSourceKeyed-SelectKey-repartition", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "keySchema" : "int" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "right2", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + }, { + "name" : "right", + "keyFormat" : { + "format" : "AVRO", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4, + "keySchema" : "int" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_SR-enabled_key_format/6.2.0_1606799825157/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_SR-enabled_key_format/6.2.0_1606799825157/topology new file mode 100644 index 000000000000..d18e17a0b803 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/multi-joins_-_table-table-table_-_SR-enabled_key_format/6.2.0_1606799825157/topology @@ -0,0 +1,125 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> KTABLE-TOSTREAM-0000000005 + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TOSTREAM-0000000005 (stores: []) + --> L_LeftSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: L_LeftSourceKeyed-SelectKey-Mapper (stores: []) + --> L_LeftSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000005 + Processor: L_LeftSourceKeyed-SelectKey-repartition-filter (stores: []) + --> L_LeftSourceKeyed-SelectKey-repartition-sink + <-- L_LeftSourceKeyed-SelectKey-Mapper + Sink: L_LeftSourceKeyed-SelectKey-repartition-sink (topic: L_LeftSourceKeyed-SelectKey-repartition) + <-- L_LeftSourceKeyed-SelectKey-repartition-filter + + Sub-topology: 1 + Source: L_LeftSourceKeyed-SelectKey-repartition-source (topics: [L_LeftSourceKeyed-SelectKey-repartition]) + --> L_LeftSourceKeyed-SelectKey + Source: L_RightSourceKeyed-SelectKey-repartition-source (topics: [L_RightSourceKeyed-SelectKey-repartition]) + --> L_RightSourceKeyed-SelectKey + Processor: L_LeftSourceKeyed-SelectKey (stores: [L_LeftSourceKeyed-Materialize]) + --> PrependAliasL_Left + <-- L_LeftSourceKeyed-SelectKey-repartition-source + Processor: L_RightSourceKeyed-SelectKey (stores: [L_RightSourceKeyed-Materialize]) + --> PrependAliasL_Right + <-- L_RightSourceKeyed-SelectKey-repartition-source + Processor: PrependAliasL_Left (stores: []) + --> KTABLE-JOINTHIS-0000000025 + <-- L_LeftSourceKeyed-SelectKey + Processor: PrependAliasL_Right (stores: []) + --> KTABLE-JOINOTHER-0000000026 + <-- L_RightSourceKeyed-SelectKey + Source: RightSourceKeyed-SelectKey-repartition-source (topics: [RightSourceKeyed-SelectKey-repartition]) + --> RightSourceKeyed-SelectKey + Processor: KTABLE-JOINOTHER-0000000026 (stores: [L_LeftSourceKeyed-Materialize]) + --> KTABLE-MERGE-0000000024 + <-- PrependAliasL_Right + Processor: KTABLE-JOINTHIS-0000000025 (stores: [L_RightSourceKeyed-Materialize]) + --> KTABLE-MERGE-0000000024 + <-- PrependAliasL_Left + Processor: RightSourceKeyed-SelectKey (stores: [RightSourceKeyed-Materialize]) + --> PrependAliasRight + <-- RightSourceKeyed-SelectKey-repartition-source + Processor: KTABLE-MERGE-0000000024 (stores: []) + --> KTABLE-JOINTHIS-0000000040 + <-- KTABLE-JOINTHIS-0000000025, KTABLE-JOINOTHER-0000000026 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000041 + <-- RightSourceKeyed-SelectKey + Processor: KTABLE-JOINOTHER-0000000041 (stores: [L_LeftSourceKeyed-Materialize, L_RightSourceKeyed-Materialize]) + --> KTABLE-MERGE-0000000039 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000040 (stores: [RightSourceKeyed-Materialize]) + --> KTABLE-MERGE-0000000039 + <-- KTABLE-MERGE-0000000024 + Processor: KTABLE-MERGE-0000000039 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000040, KTABLE-JOINOTHER-0000000041 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000043 + <-- KTABLE-MERGE-0000000039 + Processor: KTABLE-TOSTREAM-0000000043 (stores: []) + --> KSTREAM-SINK-0000000044 + <-- Project + Sink: KSTREAM-SINK-0000000044 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000043 + + Sub-topology: 2 + Source: KSTREAM-SOURCE-0000000013 (topics: [right]) + --> KTABLE-SOURCE-0000000014 + Processor: KTABLE-SOURCE-0000000014 (stores: []) + --> KTABLE-MAPVALUES-0000000015 + <-- KSTREAM-SOURCE-0000000013 + Processor: KTABLE-MAPVALUES-0000000015 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000016 + <-- KTABLE-SOURCE-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000016 (stores: []) + --> KTABLE-TOSTREAM-0000000017 + <-- KTABLE-MAPVALUES-0000000015 + Processor: KTABLE-TOSTREAM-0000000017 (stores: []) + --> L_RightSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000016 + Processor: L_RightSourceKeyed-SelectKey-Mapper (stores: []) + --> L_RightSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000017 + Processor: L_RightSourceKeyed-SelectKey-repartition-filter (stores: []) + --> L_RightSourceKeyed-SelectKey-repartition-sink + <-- L_RightSourceKeyed-SelectKey-Mapper + Sink: L_RightSourceKeyed-SelectKey-repartition-sink (topic: L_RightSourceKeyed-SelectKey-repartition) + <-- L_RightSourceKeyed-SelectKey-repartition-filter + + Sub-topology: 3 + Source: KSTREAM-SOURCE-0000000028 (topics: [right2]) + --> KTABLE-SOURCE-0000000029 + Processor: KTABLE-SOURCE-0000000029 (stores: []) + --> KTABLE-MAPVALUES-0000000030 + <-- KSTREAM-SOURCE-0000000028 + Processor: KTABLE-MAPVALUES-0000000030 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000031 + <-- KTABLE-SOURCE-0000000029 + Processor: KTABLE-TRANSFORMVALUES-0000000031 (stores: []) + --> KTABLE-TOSTREAM-0000000032 + <-- KTABLE-MAPVALUES-0000000030 + Processor: KTABLE-TOSTREAM-0000000032 (stores: []) + --> RightSourceKeyed-SelectKey-Mapper + <-- KTABLE-TRANSFORMVALUES-0000000031 + Processor: RightSourceKeyed-SelectKey-Mapper (stores: []) + --> RightSourceKeyed-SelectKey-repartition-filter + <-- KTABLE-TOSTREAM-0000000032 + Processor: RightSourceKeyed-SelectKey-repartition-filter (stores: []) + --> RightSourceKeyed-SelectKey-repartition-sink + <-- RightSourceKeyed-SelectKey-Mapper + Sink: RightSourceKeyed-SelectKey-repartition-sink (topic: RightSourceKeyed-SelectKey-repartition) + <-- RightSourceKeyed-SelectKey-repartition-filter + diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/avro.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/avro.json index 91674b742e2d..ab7407ad6b49 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/avro.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/avro.json @@ -629,9 +629,6 @@ }, { "name": "BOOLEAN - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K BOOLEAN KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -647,9 +644,6 @@ }, { "name": "BOOLEAN - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -672,9 +666,6 @@ }, { "name": "INT - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K INT KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -692,9 +683,6 @@ }, { "name": "INT - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -719,9 +707,6 @@ }, { "name": "BIGINT - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K BIGINT KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -737,9 +722,6 @@ }, { "name": "BIGINT - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -762,9 +744,6 @@ }, { "name": "DOUBLE - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K DOUBLE KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -780,9 +759,6 @@ }, { "name": "DOUBLE - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -805,9 +781,6 @@ }, { "name": "STRING - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K VARCHAR KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -823,9 +796,6 @@ }, { "name": "STRING - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -848,9 +818,6 @@ }, { "name": "DECIMAL - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K DECIMAL(4,2) KEY, foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -866,9 +833,6 @@ }, { "name": "DECIMAL - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -1071,9 +1035,6 @@ }, { "name": "INT - key - key and value inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT WITH (kafka_topic='input_topic', format='AVRO');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -1100,9 +1061,6 @@ }, { "name": "INT - key - key and anonymous value inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT WITH (kafka_topic='input_topic', format='AVRO', wrap_single_value=false);", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/formats.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/formats.json index b1ad42f081c6..73f8b8d84900 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/formats.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/formats.json @@ -276,22 +276,22 @@ "name": "unsupported key format - create as select", "statements": [ "CREATE STREAM TEST (foo VARCHAR) WITH (kafka_topic='test_topic', value_format='KAFKA');", - "CREATE STREAM COPY WITH (key_format='AVRO') AS SELECT * FROM TEST EMIT CHANGES;" + "CREATE STREAM COPY WITH (key_format='PROTOBUF') AS SELECT * FROM TEST EMIT CHANGES;" ], "expectedException": { "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The key format 'AVRO' is not currently supported" + "message": "The key format 'PROTOBUF' is not currently supported" } }, { "name": "unsupported format - create as select", "statements": [ "CREATE STREAM TEST (foo VARCHAR) WITH (kafka_topic='test_topic', value_format='KAFKA');", - "CREATE STREAM COPY WITH (format='AVRO') AS SELECT * FROM TEST EMIT CHANGES;" + "CREATE STREAM COPY WITH (format='PROTOBUF') AS SELECT * FROM TEST EMIT CHANGES;" ], "expectedException": { "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "The key format 'AVRO' is not currently supported" + "message": "The key format 'PROTOBUF' is not currently supported" } }, { diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/group-by.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/group-by.json index 02827cc4d160..16ab6bdab6df 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/group-by.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/group-by.json @@ -2315,9 +2315,6 @@ ] }, { - "properties": { - "ksql.key.format.enabled": true - }, "name": "AVRO primitive key", "statements": [ "CREATE STREAM TEST (ID INT KEY, VAL BOOLEAN) WITH (kafka_topic='test_topic', format='AVRO');", diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/joins.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/joins.json index ecfe9ee5dba5..a1d4fbbc58ba 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/joins.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/joins.json @@ -2350,9 +2350,6 @@ "CREATE STREAM S2 (ID INT KEY, VAL STRING) WITH (kafka_topic='s2', key_format='AVRO', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT S1.ID, S2.VAL FROM S1 LEFT OUTER JOIN S2 WITHIN 10 SECONDS ON S1.ID = S2.ID;" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ {"topic": "s1", "key": 10, "value": {"foo": 22}}, {"topic": "s2", "key": 10, "value": {"VAL": "hello"}} @@ -2398,9 +2395,6 @@ "CREATE STREAM S (ID INT KEY, FOO INT) WITH (kafka_topic='s', key_format='AVRO', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT S.ID, VAL FROM S JOIN T ON S.ID = T.ID;" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ {"topic": "t", "key": 10, "value": {"VAL": "hello"}}, {"topic": "s", "key": 10, "value": {"foo": 22}} @@ -2445,9 +2439,6 @@ "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, F1 varchar) WITH (kafka_topic='t2', key_format='AVRO', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT T1.id, name, f1 FROM T1 left join T2 on T1.id = T2.id;" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ {"topic": "t1", "key": 0, "value": {"NAME": "zero"}}, {"topic": "t2", "key": 0, "value": {"F1": "blah"}} @@ -2503,9 +2494,6 @@ "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, F1 varchar) WITH (kafka_topic='t2', key_format='AVRO', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT T1.id, name, f1 FROM T1 left join T2 on T1.id = T2.id;" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ {"topic": "t1", "key": 0, "value": {"NAME": "zero"}}, {"topic": "t2", "key": 0, "value": {"F1": "blah"}}, @@ -2568,9 +2556,6 @@ "CREATE STREAM S2 (VAL STRING) WITH (kafka_topic='s2', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT S1.ROWKEY, S2.VAL FROM S1 LEFT OUTER JOIN S2 WITHIN 10 SECONDS ON S1.ROWKEY = S2.ROWKEY;" ], - "properties": { - "ksql.key.format.enabled": true - }, "topics": [ { "name": "s1", @@ -2628,9 +2613,6 @@ "CREATE STREAM S2 (VAL STRING) WITH (kafka_topic='s2', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT S1.ROWKEY, S2.VAL FROM S1 LEFT OUTER JOIN S2 WITHIN 10 SECONDS ON S1.ROWKEY = S2.ROWKEY;" ], - "properties": { - "ksql.key.format.enabled": true - }, "topics": [ { "name": "s1", @@ -2692,9 +2674,6 @@ "CREATE STREAM S (FOO INT) WITH (kafka_topic='s', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT S.ROWKEY, VAL FROM S JOIN T ON S.ROWKEY = T.ROWKEY;" ], - "properties": { - "ksql.key.format.enabled": true - }, "topics": [ { "name": "t", @@ -2755,9 +2734,6 @@ "CREATE TABLE T2 (F1 varchar) WITH (kafka_topic='t2', key_format='JSON_SR', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT T1.ROWKEY, name, f1 FROM T1 left join T2 on T1.ROWKEY = T2.ROWKEY;" ], - "properties": { - "ksql.key.format.enabled": true - }, "topics": [ { "name": "t1", @@ -2831,9 +2807,6 @@ "CREATE STREAM S2 WITH (kafka_topic='s2', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT S1.ROWKEY, S2.VAL FROM S1 LEFT OUTER JOIN S2 WITHIN 10 SECONDS ON S1.ROWKEY = S2.ROWKEY;" ], - "properties": { - "ksql.key.format.enabled": true - }, "topics": [ { "name": "s1", @@ -2899,9 +2872,6 @@ "CREATE STREAM S2 (VAL STRING) WITH (kafka_topic='s2', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT ROWKEY_1, AS_VALUE(S1.ROWKEY) AS S1_KEY, AS_VALUE(S2.ROWKEY) AS S2_KEY, S2.VAL FROM S1 FULL OUTER JOIN S2 WITHIN 10 SECONDS ON S1.ROWKEY = S2.ROWKEY;" ], - "properties": { - "ksql.key.format.enabled": true - }, "topics": [ { "name": "s1", @@ -2963,9 +2933,6 @@ "CREATE TABLE T2 (ID BIGINT PRIMARY KEY, FOO INT) WITH (kafka_topic='t2', key_format='AVRO', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT T2.ID, T1.VAL FROM T1 JOIN T2 ON T1.ID = T2.ID;" ], - "properties": { - "ksql.key.format.enabled": true - }, "expectedException": { "type": "io.confluent.ksql.util.KsqlStatementException", "message": "Invalid join. Key types differ: INTEGER vs BIGINT" @@ -2977,9 +2944,6 @@ "Note: the first record on the right topic intersects with the session on the right side, but no row is output as keys must", "be an EXACT BINARY match" ], - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM S1 (ID INT KEY, V bigint) WITH (kafka_topic='left_topic', key_format='JSON_SR', value_format='JSON', WINDOW_TYPE='SESSION');", "CREATE STREAM S2 (ID INT KEY, V bigint) WITH (kafka_topic='right_topic', key_format='JSON_SR', value_format='JSON', WINDOW_TYPE='SESSION');", @@ -3034,9 +2998,6 @@ }, { "name": "matching time-windowed - SR-enabled key format", - "properties": { - "ksql.key.format.enabled": true - }, "comments": [ "Note: the two streams use a different window size. However, only the start of the window is serialized, so its possible to get a matching binary key", "This may meet users requirements, hence KSQL allows such joins", @@ -3099,9 +3060,6 @@ }, { "name": "matching time-windowed - SR-enabled key format - fails if join on non-key", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM S1 (ID INT KEY, V int) WITH (kafka_topic='left_topic', key_format='JSON_SR', value_format='JSON', WINDOW_TYPE='Hopping', WINDOW_SIZE='5 SECONDS');", "CREATE STREAM S2 (ID INT KEY, V int) WITH (kafka_topic='right_topic', key_format='JSON_SR', value_format='JSON', WINDOW_TYPE='Tumbling', WINDOW_SIZE='2 SECOND');", @@ -3119,9 +3077,6 @@ "CREATE STREAM S (ID INT KEY, FOO INT) WITH (kafka_topic='s', key_format='AVRO', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT S.ID, VAL FROM S JOIN T ON S.ID = T.VAL;" ], - "properties": { - "ksql.key.format.enabled": true - }, "expectedException": { "type": "io.confluent.ksql.util.KsqlStatementException", "message": "Cannot repartition a TABLE source. If this is a join, make sure that the criteria uses the TABLE's key column ID instead of VAL" diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/json_sr.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/json_sr.json index 4385bbe9b7a5..22122f9caa11 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/json_sr.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/json_sr.json @@ -299,9 +299,6 @@ }, { "name": "BOOLEAN - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K BOOLEAN KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -317,9 +314,6 @@ }, { "name": "BOOLEAN - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -355,9 +349,6 @@ }, { "name": "INT - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K INT KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -375,9 +366,6 @@ }, { "name": "INT - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -415,9 +403,6 @@ }, { "name": "BIGINT - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K BIGINT KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -433,9 +418,6 @@ }, { "name": "BIGINT - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -471,9 +453,6 @@ }, { "name": "DOUBLE - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K DOUBLE KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -489,9 +468,6 @@ }, { "name": "DOUBLE - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -527,9 +503,6 @@ }, { "name": "STRING - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K VARCHAR KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -545,9 +518,6 @@ }, { "name": "STRING - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -583,9 +553,6 @@ }, { "name": "DECIMAL - key - no inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (K DECIMAL(4,2) KEY, foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -601,9 +568,6 @@ }, { "name": "DECIMAL - key - inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT (foo INT) WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -829,9 +793,6 @@ }, { "name": "INT - key - key and value inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT WITH (kafka_topic='input_topic', format='JSON_SR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" @@ -858,9 +819,6 @@ }, { "name": "INT - key - key and anonymous value inference", - "properties": { - "ksql.key.format.enabled": true - }, "statements": [ "CREATE STREAM INPUT WITH (kafka_topic='input_topic', format='JSON_SR', wrap_single_value=false);", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/multi-joins.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/multi-joins.json index 68574398697c..9dff2fe0b07c 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/multi-joins.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/multi-joins.json @@ -1078,9 +1078,6 @@ "CREATE STREAM S3 (ID INT KEY, V0 bigint) WITH (kafka_topic='right2', key_format='AVRO', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S2.ID, s1.V0, s2.V0, s3.V0 FROM S1 JOIN S2 WITHIN 5 seconds ON S1.ID = S2.ID JOIN S3 WITHIN 10 seconds ON S1.ID = S3.ID;" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ {"topic": "left", "key": 0, "value": {"V0": 1}, "timestamp": 0}, {"topic": "right", "key": 0, "value": {"V0": 2}, "timestamp": 1}, @@ -1152,9 +1149,6 @@ "CREATE STREAM S3 (ID INT KEY, V0 INT) WITH (kafka_topic='right2', key_format='AVRO', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S2.ID, s1.V0, s2.V0, s3.V0 FROM S1 JOIN S2 WITHIN 5 seconds ON S1.ID = S2.ID JOIN S3 WITHIN 10 seconds ON S1.V0 = S3.V0;" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ {"topic": "left", "key": 0, "value": {"V0": 3}, "timestamp": 0}, {"topic": "right", "key": 0, "value": {"V0": 2}, "timestamp": 1}, @@ -1218,9 +1212,6 @@ "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S2.ID, s1.V0, s2.V0, t3.V0 FROM S1 JOIN S2 WITHIN 10 seconds ON S1.ID = S2.ID JOIN T3 ON S1.ID = T3.ID;" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ {"topic": "right2", "key": 0, "value": {"V0": 3}, "timestamp": 10}, {"topic": "right", "key": 0, "value": {"V0": 2}, "timestamp": 11}, @@ -1284,9 +1275,6 @@ "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', key_format='JSON_SR', value_format='JSON');", "CREATE STREAM OUTPUT as SELECT S2.ID, s1.V0, s2.V0, t3.V0 FROM S1 JOIN S2 WITHIN 10 seconds ON S1.ID = S2.ID JOIN T3 ON S1.V0 = T3.ID;" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ {"topic": "right2", "key": 1, "value": {"V0": 3}, "timestamp": 10}, {"topic": "right", "key": 0, "value": {"V0": 2}, "timestamp": 11}, @@ -1350,9 +1338,6 @@ "CREATE TABLE T3 (ID INT PRIMARY KEY, V0 bigint) WITH (kafka_topic='right2', key_format='AVRO', value_format='JSON');", "CREATE TABLE OUTPUT as SELECT ROWKEY AS ID, T1.V0, T2.V0, T3.V0 FROM T1 JOIN T2 ON T1.ID = T2.ID FULL JOIN T3 ON T1.ID = T3.ID;" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ {"topic": "left", "key": 0, "value": {"V0": 1}, "timestamp": 0}, {"topic": "right2", "key": 0, "value": {"V0": 3}, "timestamp": 2}, diff --git a/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/insert-values.json b/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/insert-values.json index 60f9931ed7f7..4a61ae95df41 100644 --- a/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/insert-values.json +++ b/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/insert-values.json @@ -390,9 +390,6 @@ "CREATE STREAM TEST (K STRING KEY, ID INT) WITH (kafka_topic='test_topic', format='JSON');", "INSERT INTO TEST (ID, ROWTIME, K) VALUES (10, 1234, 'key');" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ ], "outputs": [ @@ -405,9 +402,6 @@ "CREATE STREAM TEST (K STRING KEY, ID INT) WITH (kafka_topic='test_topic', format='AVRO');", "INSERT INTO TEST (ID, ROWTIME, K) VALUES (10, 1234, 'key');" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ ], "outputs": [ diff --git a/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/pull-queries-against-materialized-aggregates.json b/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/pull-queries-against-materialized-aggregates.json index d2e661490c81..b44166db1ed5 100644 --- a/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/pull-queries-against-materialized-aggregates.json +++ b/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/pull-queries-against-materialized-aggregates.json @@ -1509,9 +1509,6 @@ "SELECT * FROM AGGREGATE WHERE ID=10;", "SELECT * FROM AGGREGATE WHERE ID=123369;" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ {"topic": "test_topic", "timestamp": 12345, "key": 11, "value": {}}, {"topic": "test_topic", "timestamp": 12365, "key": 10, "value": {}} diff --git a/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/push-queries.json b/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/push-queries.json index 724076d5c1a1..4ae9c1411388 100644 --- a/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/push-queries.json +++ b/ksqldb-functional-tests/src/test/resources/rest-query-validation-tests/push-queries.json @@ -736,9 +736,6 @@ "CREATE STREAM INPUT (K INT KEY, ID INT) WITH (kafka_topic='test_topic', format='JSON');", "SELECT * FROM INPUT EMIT CHANGES LIMIT 2;" ], - "properties": { - "ksql.key.format.enabled": true - }, "inputs": [ {"topic": "test_topic", "timestamp": 12345, "key": 11, "value": {"id": 100}}, {"topic": "test_topic", "timestamp": 12365, "key": 11, "value": {"id": 101}} diff --git a/ksqldb-functional-tests/src/test/resources/sql-tests/test.sql b/ksqldb-functional-tests/src/test/resources/sql-tests/test.sql index 30a9e5148248..460344572e93 100644 --- a/ksqldb-functional-tests/src/test/resources/sql-tests/test.sql +++ b/ksqldb-functional-tests/src/test/resources/sql-tests/test.sql @@ -170,8 +170,6 @@ ASSERT STREAM bar (id INT KEY, col1 INT) WITH (kafka_topic='BAZ', value_format=' --@expected.error: io.confluent.ksql.util.KsqlException --@expected.message: Expected key format does not match actual for source BAR ---------------------------------------------------------------------------------------------------- -SET 'ksql.key.format.enabled' = 'true'; - CREATE STREAM foo (id INT KEY, col1 INT) WITH (kafka_topic='foo', key_format='KAFKA', value_format='JSON'); CREATE STREAM bar AS SELECT * FROM foo; @@ -194,8 +192,6 @@ ASSERT STREAM bar (id INT KEY, col1 INT) WITH (kafka_topic='BAR', value_format=' --@expected.error: io.confluent.ksql.util.KsqlException --@expected.message: Expected value format does not match actual for source BAR ---------------------------------------------------------------------------------------------------- -SET 'ksql.key.format.enabled' = 'true'; - CREATE STREAM foo (id INT KEY, col1 INT) WITH (kafka_topic='foo', format='KAFKA'); CREATE STREAM bar AS SELECT * FROM foo; @@ -204,8 +200,6 @@ ASSERT STREAM bar (id INT KEY, col1 INT) WITH (kafka_topic='BAR', key_format='KA ---------------------------------------------------------------------------------------------------- --@test: assert stream with explicit format ---------------------------------------------------------------------------------------------------- -SET 'ksql.key.format.enabled' = 'true'; - CREATE STREAM foo (id INT KEY, col1 INT) WITH (kafka_topic='foo', format='KAFKA'); CREATE STREAM bar AS SELECT * FROM foo; @@ -214,8 +208,6 @@ ASSERT STREAM bar (id INT KEY, col1 INT) WITH (kafka_topic='BAR', key_format='KA ---------------------------------------------------------------------------------------------------- --@test: assert stream with explicit expected format ---------------------------------------------------------------------------------------------------- -SET 'ksql.key.format.enabled' = 'true'; - CREATE STREAM foo (id INT KEY, col1 INT) WITH (kafka_topic='foo', key_format='KAFKA', value_format='KAFKA'); CREATE STREAM bar AS SELECT * FROM foo; diff --git a/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/integration/ClusterTerminationTest.java b/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/integration/ClusterTerminationTest.java index ba8359bd4abf..4a19793fae45 100644 --- a/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/integration/ClusterTerminationTest.java +++ b/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/integration/ClusterTerminationTest.java @@ -61,7 +61,6 @@ public class ClusterTerminationTest { .builder(TEST_HARNESS::kafkaBootstrapServers) .withStaticServiceContext(TEST_HARNESS::getServiceContext) .withProperty(KsqlConfig.SCHEMA_REGISTRY_URL_PROPERTY, "http://foo:8080") - .withProperty(KsqlConfig.KSQL_KEY_FORMAT_ENABLED, true) .build(); @ClassRule diff --git a/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/integration/KsqlResourceFunctionalTest.java b/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/integration/KsqlResourceFunctionalTest.java index 0c2f401f1942..a24a2fa60287 100644 --- a/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/integration/KsqlResourceFunctionalTest.java +++ b/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/integration/KsqlResourceFunctionalTest.java @@ -77,7 +77,6 @@ public class KsqlResourceFunctionalTest { .builder(TEST_HARNESS::kafkaBootstrapServers) .withStaticServiceContext(TEST_HARNESS::getServiceContext) .withProperty(KsqlConfig.SCHEMA_REGISTRY_URL_PROPERTY, "http://foo:8080") - .withProperty(KsqlConfig.KSQL_KEY_FORMAT_ENABLED, true) .build(); @ClassRule diff --git a/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/server/KsqlServerMainTest.java b/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/server/KsqlServerMainTest.java index eb18a36f788b..6fff022cbf31 100644 --- a/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/server/KsqlServerMainTest.java +++ b/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/server/KsqlServerMainTest.java @@ -263,7 +263,7 @@ public void shouldFailOnInvalidDefaultValueFormat() { public void shouldFailOnUnsupportedDefaultKeyFormat() { // Given: final KsqlConfig config = configWith(ImmutableMap.of( - KsqlConfig.KSQL_DEFAULT_KEY_FORMAT_CONFIG, "avro" + KsqlConfig.KSQL_DEFAULT_KEY_FORMAT_CONFIG, "protobuf" )); // When: @@ -276,7 +276,7 @@ public void shouldFailOnUnsupportedDefaultKeyFormat() { assertThat(e.getMessage(), containsString( "Invalid value for config '" + KsqlConfig.KSQL_DEFAULT_KEY_FORMAT_CONFIG + "': " + "The supplied format is not currently supported as a key format. " - + "Format: 'avro'.")); + + "Format: 'protobuf'.")); } private static KsqlConfig configWith(final Map additionalConfigs) { diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/KeyFormatUtils.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/KeyFormatUtils.java index f68414bf0541..621d6ea834d5 100644 --- a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/KeyFormatUtils.java +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/KeyFormatUtils.java @@ -25,13 +25,13 @@ public final class KeyFormatUtils { FormatFactory.NONE, FormatFactory.KAFKA, FormatFactory.DELIMITED, - FormatFactory.JSON + FormatFactory.JSON, + FormatFactory.AVRO, + FormatFactory.JSON_SR ); private static final List KEY_FORMATS_UNDER_DEVELOPMENT = ImmutableList.of( - FormatFactory.AVRO, - FormatFactory.PROTOBUF, - FormatFactory.JSON_SR + FormatFactory.PROTOBUF ); /**