From a9668de461566f401d79666c4d5fff9ff0c255a6 Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Wed, 28 Jul 2021 18:00:14 -0700 Subject: [PATCH] feat: allow expressions on left table columns in FK-joins (#7904) Currently, FK-join require to use a plain column reference for the left join expression. This PR lifts this restriction and allows for actual expression in the left join expression. --- .../ksql/planner/LogicalPlanner.java | 83 ++-- .../confluent/ksql/planner/plan/JoinNode.java | 41 +- .../ksql/structured/SchemaKTable.java | 16 +- .../plan/ForeignKeyTableTableJoin.java | 44 +- .../plan/ForeignKeyTableTableJoinTest.java | 105 ++++- .../7.1.0_1627438334984/plan.json | 249 +++++++++++ .../7.1.0_1627438334984/spec.json | 273 ++++++++++++ .../7.1.0_1627438334984/topology | 67 +++ .../7.1.0_1627438334610/plan.json | 277 ++++++++++++ .../7.1.0_1627438334610/spec.json | 396 ++++++++++++++++++ .../7.1.0_1627438334610/topology | 67 +++ .../7.1.0_1627438333848/plan.json | 249 +++++++++++ .../7.1.0_1627438333848/spec.json | 273 ++++++++++++ .../7.1.0_1627438333848/topology | 67 +++ .../7.1.0_1627438334234/plan.json | 246 +++++++++++ .../7.1.0_1627438334234/spec.json | 269 ++++++++++++ .../7.1.0_1627438334234/topology | 67 +++ .../7.1.0_1627438332148/plan.json | 246 +++++++++++ .../7.1.0_1627438332148/spec.json | 265 ++++++++++++ .../7.1.0_1627438332148/topology | 67 +++ .../7.1.0_1627438332551/plan.json | 246 +++++++++++ .../7.1.0_1627438332551/spec.json | 213 ++++++++++ .../7.1.0_1627438332551/topology | 67 +++ .../7.1.0_1627438331306/plan.json | 246 +++++++++++ .../7.1.0_1627438331306/spec.json | 268 ++++++++++++ .../7.1.0_1627438331306/topology | 67 +++ .../7.1.0_1627438331684/plan.json | 246 +++++++++++ .../7.1.0_1627438331684/spec.json | 321 ++++++++++++++ .../7.1.0_1627438331684/topology | 67 +++ .../7.1.0_1627438328984/plan.json | 249 +++++++++++ .../7.1.0_1627438328984/spec.json | 302 +++++++++++++ .../7.1.0_1627438328984/topology | 67 +++ .../7.1.0_1627438328037/plan.json | 246 +++++++++++ .../7.1.0_1627438328037/spec.json | 268 ++++++++++++ .../7.1.0_1627438328037/topology | 67 +++ .../7.1.0_1627438329904/plan.json | 246 +++++++++++ .../7.1.0_1627438329904/spec.json | 268 ++++++++++++ .../7.1.0_1627438329904/topology | 67 +++ .../7.1.0_1627438329411/plan.json | 249 +++++++++++ .../7.1.0_1627438329411/spec.json | 373 +++++++++++++++++ .../7.1.0_1627438329411/topology | 67 +++ .../7.1.0_1627438328502/plan.json | 246 +++++++++++ .../7.1.0_1627438328502/spec.json | 321 ++++++++++++++ .../7.1.0_1627438328502/topology | 67 +++ .../7.1.0_1627438330284/plan.json | 246 +++++++++++ .../7.1.0_1627438330284/spec.json | 321 ++++++++++++++ .../7.1.0_1627438330284/topology | 67 +++ .../7.1.0_1627438330922/plan.json | 246 +++++++++++ .../7.1.0_1627438330922/spec.json | 268 ++++++++++++ .../7.1.0_1627438330922/topology | 67 +++ .../7.1.0_1627505730914/plan.json | 246 +++++++++++ .../7.1.0_1627505730914/spec.json | 268 ++++++++++++ .../7.1.0_1627505730914/topology | 67 +++ .../7.1.0_1627505730530/plan.json | 246 +++++++++++ .../7.1.0_1627505730530/spec.json | 268 ++++++++++++ .../7.1.0_1627505730530/topology | 67 +++ .../7.1.0_1627438333496/plan.json | 246 +++++++++++ .../7.1.0_1627438333496/spec.json | 245 +++++++++++ .../7.1.0_1627438333496/topology | 67 +++ .../7.1.0_1627438333184/plan.json | 246 +++++++++++ .../7.1.0_1627438333184/spec.json | 211 ++++++++++ .../7.1.0_1627438333184/topology | 67 +++ .../7.1.0_1627438332834/plan.json | 246 +++++++++++ .../7.1.0_1627438332834/spec.json | 237 +++++++++++ .../7.1.0_1627438332834/topology | 67 +++ .../7.1.0_1627438335608/plan.json | 318 ++++++++++++++ .../7.1.0_1627438335608/spec.json | 328 +++++++++++++++ .../7.1.0_1627438335608/topology | 90 ++++ .../7.1.0_1627438336446/plan.json | 318 ++++++++++++++ .../7.1.0_1627438336446/spec.json | 328 +++++++++++++++ .../7.1.0_1627438336446/topology | 90 ++++ .../7.1.0_1627438337258/plan.json | 318 ++++++++++++++ .../7.1.0_1627438337258/spec.json | 331 +++++++++++++++ .../7.1.0_1627438337258/topology | 90 ++++ .../7.1.0_1627438336855/plan.json | 318 ++++++++++++++ .../7.1.0_1627438336855/spec.json | 334 +++++++++++++++ .../7.1.0_1627438336855/topology | 90 ++++ .../7.1.0_1627438337654/plan.json | 318 ++++++++++++++ .../7.1.0_1627438337654/spec.json | 328 +++++++++++++++ .../7.1.0_1627438337654/topology | 90 ++++ .../7.1.0_1627438336048/plan.json | 318 ++++++++++++++ .../7.1.0_1627438336048/spec.json | 322 ++++++++++++++ .../7.1.0_1627438336048/topology | 90 ++++ .../query-validation-tests/fk-join.json | 110 ++++- .../resources/ksql-plan-schema/schema.json | 5 +- .../streams/ExecutionStepFactory.java | 8 +- .../streams/ForeignKeyJoinParamsFactory.java | 24 +- .../ForeignKeyTableTableJoinBuilder.java | 36 +- .../execution/streams/KsqlKeyExtractor.java | 41 +- .../ForeignKeyJoinParamsFactoryTest.java | 25 +- .../ForeignKeyTableTableJoinBuilderTest.java | 31 +- 91 files changed, 16777 insertions(+), 142 deletions(-) create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/topology diff --git a/ksqldb-engine/src/main/java/io/confluent/ksql/planner/LogicalPlanner.java b/ksqldb-engine/src/main/java/io/confluent/ksql/planner/LogicalPlanner.java index f8f0e5e92bb5..3826c56c8ebb 100644 --- a/ksqldb-engine/src/main/java/io/confluent/ksql/planner/LogicalPlanner.java +++ b/ksqldb-engine/src/main/java/io/confluent/ksql/planner/LogicalPlanner.java @@ -28,6 +28,7 @@ import io.confluent.ksql.analyzer.RewrittenAnalysis; import io.confluent.ksql.engine.rewrite.ExpressionTreeRewriter; import io.confluent.ksql.engine.rewrite.ExpressionTreeRewriter.Context; +import io.confluent.ksql.execution.codegen.CodeGenRunner; import io.confluent.ksql.execution.ddl.commands.KsqlTopic; import io.confluent.ksql.execution.expression.tree.ColumnReferenceExp; import io.confluent.ksql.execution.expression.tree.Expression; @@ -39,6 +40,7 @@ import io.confluent.ksql.execution.streams.PartitionByParamsFactory; import io.confluent.ksql.execution.streams.timestamp.TimestampExtractionPolicyFactory; import io.confluent.ksql.execution.timestamp.TimestampColumn; +import io.confluent.ksql.execution.transform.ExpressionEvaluator; import io.confluent.ksql.execution.util.ExpressionTypeManager; import io.confluent.ksql.execution.windows.KsqlWindowExpression; import io.confluent.ksql.function.udf.AsValue; @@ -597,11 +599,11 @@ private JoinNode buildJoin(final Join root, final String prefix, final boolean i ); } - final Optional fkColumnName = + final Optional fkExpression = verifyJoin(root.getInfo(), preRepartitionLeft, preRepartitionRight); - final JoinKey joinKey = fkColumnName - .map(columnReferenceExp -> buildForeignJoinKey(root, fkColumnName.get())) + final JoinKey joinKey = fkExpression + .map(columnReferenceExp -> buildForeignJoinKey(root, fkExpression.get())) .orElseGet(() -> buildJoinKey(root)); final PlanNode left = prepareSourceForJoin( @@ -609,14 +611,14 @@ private JoinNode buildJoin(final Join root, final String prefix, final boolean i preRepartitionLeft, prefix + "Left", root.getInfo().getLeftJoinExpression(), - fkColumnName.isPresent() + fkExpression.isPresent() ); final PlanNode right = prepareSourceForJoin( root.getRight(), preRepartitionRight, prefix + "Right", root.getInfo().getRightJoinExpression(), - fkColumnName.isPresent() + fkExpression.isPresent() ); return new JoinNode( @@ -634,7 +636,7 @@ private JoinNode buildJoin(final Join root, final String prefix, final boolean i /** * @return the foreign key column if this is a foreign key join */ - private Optional verifyJoin( + private Optional verifyJoin( final JoinInfo joinInfo, final PlanNode leftNode, final PlanNode rightNode @@ -721,7 +723,7 @@ private static void verifyStreamTableJoin(final JoinInfo joinInfo, final PlanNod } } - private Optional verifyForeignKeyJoin( + private Optional verifyForeignKeyJoin( final JoinInfo joinInfo, final PlanNode leftNode, final PlanNode rightNode @@ -755,25 +757,34 @@ private Optional verifyForeignKeyJoin( )); } - if (!(leftExpression instanceof ColumnReferenceExp)) { - throw new KsqlException(String.format( - "Invalid join condition:" - + " foreign-key table-table joins with expressions are not supported yet." - + " Got %s = %s.", - joinInfo.getFlippedLeftJoinExpression(), - joinInfo.getFlippedRightJoinExpression() - )); - } - - // we need to extend this to support expressions later on - final ColumnReferenceExp fkColumnReference = (ColumnReferenceExp) leftExpression; + final CodeGenRunner codeGenRunner = new CodeGenRunner( + leftNode.getSchema(), + ksqlConfig, + metaStore + ); - final SqlType fkColumnType = - leftNode.getSchema().findColumn(fkColumnReference.getColumnName()).get().type(); + final VisitParentExpressionVisitor, Context> unqualifiedRewritter = + new VisitParentExpressionVisitor, Context>(Optional.empty()) { + @Override + public Optional visitQualifiedColumnReference( + final QualifiedColumnReferenceExp node, + final Context ctx + ) { + return Optional.of(new UnqualifiedColumnReferenceExp(node.getColumnName())); + } + }; + + final Expression leftExpressionUnqualified = + ExpressionTreeRewriter.rewriteWith(unqualifiedRewritter::process, leftExpression); + final ExpressionEvaluator expressionEvaluator = codeGenRunner.buildCodeGenFromParseTree( + leftExpressionUnqualified, + "Left Join Expression" + ); + final SqlType fkType = expressionEvaluator.getExpressionType(); final SqlType rightKeyType = Iterables.getOnlyElement(rightNode.getSchema().key()).type(); verifyJoinConditionTypes( - fkColumnType, + fkType, rightKeyType, leftExpression, rightExpression, @@ -786,7 +797,7 @@ private Optional verifyForeignKeyJoin( ); } - return Optional.of(fkColumnReference); + return Optional.of(leftExpression); } private static boolean joinOnNonKeyAttribute( @@ -872,7 +883,8 @@ private static boolean isInnerNode(final PlanNode node) { throw new IllegalStateException("Unknown node type: " + node.getClass().getName()); } - private JoinKey buildForeignJoinKey(final Join join, final ColumnReferenceExp columnRef) { + private JoinKey buildForeignJoinKey(final Join join, + final Expression foreignKeyExpression) { final AliasedDataSource leftSource = join.getInfo().getLeftSource(); final SourceName alias = leftSource.getAlias(); final List leftSourceKeys = @@ -880,13 +892,24 @@ private JoinKey buildForeignJoinKey(final Join join, final ColumnReferenceExp co .map(c -> new QualifiedColumnReferenceExp(alias, c.name())) .collect(Collectors.toList()); - final ColumnName foreignKeyColumnName = columnRef.maybeQualifier().isPresent() - ? ColumnNames.generatedJoinColumnAlias( - columnRef.maybeQualifier().get(), - columnRef.getColumnName()) - : columnRef.getColumnName(); + final VisitParentExpressionVisitor, Context> aliasRewritter = + new VisitParentExpressionVisitor, Context>(Optional.empty()) { + @Override + public Optional visitQualifiedColumnReference( + final QualifiedColumnReferenceExp node, + final Context ctx + ) { + return Optional.of(new UnqualifiedColumnReferenceExp( + ColumnNames.generatedJoinColumnAlias(node.getQualifier(), node.getColumnName()) + )); + } + }; + + final Expression aliasedForeignKeyExpression = + ExpressionTreeRewriter.rewriteWith(aliasRewritter::process, foreignKeyExpression); + - return JoinKey.foreignKeyColumn(foreignKeyColumnName, leftSourceKeys); + return JoinKey.foreignKeyColumn(aliasedForeignKeyExpression, leftSourceKeys); } private static void verifyJoinConditionTypes( diff --git a/ksqldb-engine/src/main/java/io/confluent/ksql/planner/plan/JoinNode.java b/ksqldb-engine/src/main/java/io/confluent/ksql/planner/plan/JoinNode.java index ae05e20833fa..4788a3ca162f 100644 --- a/ksqldb-engine/src/main/java/io/confluent/ksql/planner/plan/JoinNode.java +++ b/ksqldb-engine/src/main/java/io/confluent/ksql/planner/plan/JoinNode.java @@ -535,7 +535,8 @@ public SchemaKTable join() { rightTable, ((ForeignJoinKey) joinKey).getForeignKeyColumn(), contextStacker, - valueFormatInfo + valueFormatInfo, + ((ForeignJoinKey) joinKey).getForeignKeyExpression() ); } else { return leftTable.leftJoin( @@ -550,7 +551,8 @@ public SchemaKTable join() { rightTable, ((ForeignJoinKey) joinKey).getForeignKeyColumn(), contextStacker, - valueFormatInfo + valueFormatInfo, + ((ForeignJoinKey) joinKey).getForeignKeyExpression() ); } else { return leftTable.innerJoin( @@ -643,10 +645,10 @@ static JoinKey syntheticColumn() { } static JoinKey foreignKeyColumn( - final ColumnName foreignKeyColumn, + final Expression foreignKeyExpression, final Collection viableKeyColumns ) { - return ForeignJoinKey.of(foreignKeyColumn, viableKeyColumns); + return ForeignJoinKey.of(foreignKeyExpression, viableKeyColumns); } /** @@ -806,17 +808,34 @@ public JoinKey rewriteWith( } private static final class ForeignJoinKey implements JoinKey { - private final ColumnName foreignKeyColumn; + private final Optional foreignKeyColumn; + private final Optional foreignKeyExpression; private final ImmutableList leftSourceKeyColumns; static JoinKey of(final ColumnName foreignKeyColumn, final Collection leftSourceKeyColumns) { - return new ForeignJoinKey(foreignKeyColumn, leftSourceKeyColumns); + return new ForeignJoinKey( + Optional.of(foreignKeyColumn), + Optional.empty(), + leftSourceKeyColumns + ); } - private ForeignJoinKey(final ColumnName foreignKeyColumn, + static JoinKey of(final Expression foreignKeyExpression, + final Collection leftSourceKeyColumns) { + return new ForeignJoinKey( + Optional.empty(), + Optional.of(foreignKeyExpression), + leftSourceKeyColumns + ); + } + + private ForeignJoinKey(final Optional foreignKeyColumn, + final Optional foreignKeyExpression, final Collection viableKeyColumns) { this.foreignKeyColumn = requireNonNull(foreignKeyColumn, "foreignKeyColumn"); + this.foreignKeyExpression = + requireNonNull(foreignKeyExpression, "foreignKeyExpression"); this.leftSourceKeyColumns = ImmutableList .copyOf(requireNonNull(viableKeyColumns, "viableKeyColumns")); } @@ -853,11 +872,15 @@ public JoinKey rewriteWith( .map(e -> ExpressionTreeRewriter.rewriteWith(plugin, e)) .collect(Collectors.toList()); - return new ForeignJoinKey(foreignKeyColumn, rewrittenViable); + return new ForeignJoinKey(Optional.empty(), foreignKeyExpression, rewrittenViable); } - public ColumnName getForeignKeyColumn() { + public Optional getForeignKeyColumn() { return foreignKeyColumn; } + + public Optional getForeignKeyExpression() { + return foreignKeyExpression; + } } } diff --git a/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKTable.java b/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKTable.java index 2e9c05bdb07f..a1522da20d28 100644 --- a/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKTable.java +++ b/ksqldb-engine/src/main/java/io/confluent/ksql/structured/SchemaKTable.java @@ -321,9 +321,10 @@ public SchemaKTable outerJoin( public SchemaKTable foreignKeyInnerJoin( final SchemaKTable schemaKTable, - final ColumnName leftJoinColumnName, + final Optional leftJoinColumnName, final Stacker contextStacker, - final FormatInfo valueFormatInfo + final FormatInfo valueFormatInfo, + final Optional leftJoinExpression ) { final ForeignKeyTableTableJoin step = ExecutionStepFactory.foreignKeyTableTableJoin( @@ -332,7 +333,8 @@ public SchemaKTable foreignKeyInnerJoin( leftJoinColumnName, InternalFormats.of(keyFormat, valueFormatInfo), sourceTableStep, - schemaKTable.getSourceTableStep() + schemaKTable.getSourceTableStep(), + leftJoinExpression ); return new SchemaKTable<>( @@ -346,9 +348,10 @@ public SchemaKTable foreignKeyInnerJoin( public SchemaKTable foreignKeyLeftJoin( final SchemaKTable schemaKTable, - final ColumnName leftJoinColumnName, + final Optional leftJoinColumnName, final Stacker contextStacker, - final FormatInfo valueFormatInfo + final FormatInfo valueFormatInfo, + final Optional leftJoinExpression ) { final ForeignKeyTableTableJoin step = ExecutionStepFactory.foreignKeyTableTableJoin( @@ -357,7 +360,8 @@ public SchemaKTable foreignKeyLeftJoin( leftJoinColumnName, InternalFormats.of(keyFormat, valueFormatInfo), sourceTableStep, - schemaKTable.getSourceTableStep() + schemaKTable.getSourceTableStep(), + leftJoinExpression ); return new SchemaKTable<>( diff --git a/ksqldb-execution/src/main/java/io/confluent/ksql/execution/plan/ForeignKeyTableTableJoin.java b/ksqldb-execution/src/main/java/io/confluent/ksql/execution/plan/ForeignKeyTableTableJoin.java index 3f849494a011..9e53096c6e83 100644 --- a/ksqldb-execution/src/main/java/io/confluent/ksql/execution/plan/ForeignKeyTableTableJoin.java +++ b/ksqldb-execution/src/main/java/io/confluent/ksql/execution/plan/ForeignKeyTableTableJoin.java @@ -21,9 +21,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.Immutable; +import io.confluent.ksql.execution.expression.tree.Expression; import io.confluent.ksql.name.ColumnName; import java.util.List; import java.util.Objects; +import java.util.Optional; @Immutable public class ForeignKeyTableTableJoin @@ -31,7 +33,8 @@ public class ForeignKeyTableTableJoin private final ExecutionStepPropertiesV1 properties; private final JoinType joinType; - private final ColumnName leftJoinColumnName; + private final Optional leftJoinColumnName; + private final Optional leftJoinExpression; private final Formats formats; private final ExecutionStep> leftSource; private final ExecutionStep> rightSource; @@ -42,14 +45,16 @@ public ForeignKeyTableTableJoin( final ExecutionStepPropertiesV1 props, @JsonProperty(value = "joinType", required = true) final JoinType joinType, - @JsonProperty(value = "leftJoinColumnName", required = true) - final ColumnName leftJoinColumnName, + @JsonProperty(value = "leftJoinColumnName") + final Optional leftJoinColumnName, @JsonProperty(value = "formats", required = true) final Formats formats, @JsonProperty(value = "leftSource", required = true) final ExecutionStep> leftSource, @JsonProperty(value = "rightSource", required = true) - final ExecutionStep> rightSource + final ExecutionStep> rightSource, + @JsonProperty(value = "leftJoinExpression") + final Optional leftJoinExpression ) { this.properties = requireNonNull(props, "props"); this.joinType = requireNonNull(joinType, "joinType"); @@ -60,6 +65,17 @@ public ForeignKeyTableTableJoin( this.formats = requireNonNull(formats, "formats"); this.leftSource = requireNonNull(leftSource, "leftSource"); this.rightSource = requireNonNull(rightSource, "rightSource"); + this.leftJoinExpression = requireNonNull(leftJoinExpression, "leftJoinExpression"); + if (!leftJoinColumnName.isPresent() && !leftJoinExpression.isPresent()) { + throw new IllegalArgumentException( + "Either leftJoinColumnName or leftJoinExpression must be provided." + ); + } + if (leftJoinColumnName.isPresent() && leftJoinExpression.isPresent()) { + throw new IllegalArgumentException( + "Either leftJoinColumnName or leftJoinExpression must be empty." + ); + } } @Override @@ -85,10 +101,14 @@ public JoinType getJoinType() { return joinType; } - public ColumnName getLeftJoinColumnName() { + public Optional getLeftJoinColumnName() { return leftJoinColumnName; } + public Optional getLeftJoinExpression() { + return leftJoinExpression; + } + public Formats getFormats() { return formats; } @@ -117,11 +137,21 @@ public boolean equals(final Object o) { && Objects.equals(leftJoinColumnName, that.leftJoinColumnName) && Objects.equals(formats, that.formats) && Objects.equals(leftSource, that.leftSource) - && Objects.equals(rightSource, that.rightSource); + && Objects.equals(rightSource, that.rightSource) + && Objects.equals(leftJoinExpression, that.leftJoinExpression); + } @Override public int hashCode() { - return Objects.hash(properties, joinType, leftJoinColumnName, formats, leftSource, rightSource); + return Objects.hash( + properties, + joinType, + leftJoinColumnName, + formats, + leftSource, + rightSource, + leftJoinExpression + ); } } diff --git a/ksqldb-execution/src/test/java/io/confluent/ksql/execution/plan/ForeignKeyTableTableJoinTest.java b/ksqldb-execution/src/test/java/io/confluent/ksql/execution/plan/ForeignKeyTableTableJoinTest.java index aed4a4df4c2c..c6b1d0891a83 100644 --- a/ksqldb-execution/src/test/java/io/confluent/ksql/execution/plan/ForeignKeyTableTableJoinTest.java +++ b/ksqldb-execution/src/test/java/io/confluent/ksql/execution/plan/ForeignKeyTableTableJoinTest.java @@ -16,10 +16,19 @@ import static io.confluent.ksql.execution.plan.JoinType.INNER; import static io.confluent.ksql.execution.plan.JoinType.LEFT; +import static io.confluent.ksql.execution.plan.JoinType.OUTER; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThrows; import com.google.common.testing.EqualsTester; +import io.confluent.ksql.execution.expression.tree.Cast; +import io.confluent.ksql.execution.expression.tree.Expression; +import io.confluent.ksql.execution.expression.tree.Type; +import io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp; import io.confluent.ksql.name.ColumnName; -import io.confluent.ksql.serde.FormatInfo; +import io.confluent.ksql.schema.ksql.types.SqlPrimitiveType; +import java.util.Optional; import org.apache.kafka.connect.data.Struct; import org.junit.Test; import org.junit.runner.RunWith; @@ -29,8 +38,18 @@ @RunWith(MockitoJUnitRunner.class) public class ForeignKeyTableTableJoinTest { - private static final ColumnName JOIN_COLUMN_NAME = ColumnName.of("Bob"); - private static final ColumnName JOIN_COLUMN_NAME_2 = ColumnName.of("Vic"); + private static final Optional JOIN_COLUMN_NAME = Optional.of(ColumnName.of("Bob")); + private static final Optional JOIN_COLUMN_NAME_2 = Optional.of(ColumnName.of("Vic")); + private static final Optional JOIN_EXPRESSION = Optional.of( + new Cast( + new UnqualifiedColumnReferenceExp(Optional.empty(), ColumnName.of("Bob")), + new Type(SqlPrimitiveType.of("INT")) + )); + private static final Optional JOIN_EXPRESSION_2 = Optional.of( + new Cast( + new UnqualifiedColumnReferenceExp(Optional.empty(), ColumnName.of("Vic")), + new Type(SqlPrimitiveType.of("VARCHAR")) + )); @Mock private ExecutionStepPropertiesV1 props1; @@ -49,31 +68,91 @@ public class ForeignKeyTableTableJoinTest { @Mock private Formats formats2; + @Test + public void shouldNotAllowOuterJoin() { + final IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> new ForeignKeyTableTableJoin<>(props1, OUTER, JOIN_COLUMN_NAME, formats1, left1, right1, Optional.empty()) + ); + + assertThat(error.getMessage(), is("OUTER join not supported.")); + } + + @Test + public void shouldNotAllowEmptyColumnNameAndEmptyExpression() { + final IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> new ForeignKeyTableTableJoin<>(props1, INNER, Optional.empty(), formats1, left1, right1, Optional.empty()) + ); + + assertThat(error.getMessage(), is("Either leftJoinColumnName or leftJoinExpression must be provided.")); + } + + @Test + public void shouldNotAllowColumnNameAndExpression() { + final IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME, formats1, left1, right1, JOIN_EXPRESSION) + ); + + assertThat(error.getMessage(), is("Either leftJoinColumnName or leftJoinExpression must be empty.")); + } + + @SuppressWarnings("UnstableApiUsage") + @Test + public void shouldImplementEqualsColumName() { + new EqualsTester() + .addEqualityGroup( + new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME, formats1, left1, right1, Optional.empty()), + new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME, formats1, left1, right1, Optional.empty()) + ) + .addEqualityGroup( + new ForeignKeyTableTableJoin<>(props2, INNER, JOIN_COLUMN_NAME, formats1, left1, right1, Optional.empty()) + ) + .addEqualityGroup( + new ForeignKeyTableTableJoin<>(props1, LEFT, JOIN_COLUMN_NAME, formats1, left1, right1, Optional.empty()) + ) + .addEqualityGroup( + new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME_2, formats1, left1, right1, Optional.empty()) + ) + .addEqualityGroup( + new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME, formats2, left1, right1, Optional.empty()) + ) + .addEqualityGroup( + new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME, formats1, left2, right1, Optional.empty()) + ) + .addEqualityGroup( + new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME, formats1, left1, right2, Optional.empty()) + ) + .testEquals(); + } + @SuppressWarnings("UnstableApiUsage") @Test - public void shouldImplementEquals() { + public void shouldImplementEqualsExpression() { new EqualsTester() .addEqualityGroup( - new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME, formats1, left1, right1), - new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME, formats1, left1, right1) + new ForeignKeyTableTableJoin<>(props1, INNER, Optional.empty(), formats1, left1, right1, JOIN_EXPRESSION), + new ForeignKeyTableTableJoin<>(props1, INNER, Optional.empty(), formats1, left1, right1, JOIN_EXPRESSION) ) .addEqualityGroup( - new ForeignKeyTableTableJoin<>(props2, INNER, JOIN_COLUMN_NAME, formats1, left1, right1) + new ForeignKeyTableTableJoin<>(props2, INNER, Optional.empty(), formats1, left1, right1, JOIN_EXPRESSION) ) .addEqualityGroup( - new ForeignKeyTableTableJoin<>(props1, LEFT, JOIN_COLUMN_NAME, formats1, left1, right1) + new ForeignKeyTableTableJoin<>(props1, LEFT, Optional.empty(), formats1, left1, right1, JOIN_EXPRESSION) ) .addEqualityGroup( - new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME_2, formats1, left1, right1) + new ForeignKeyTableTableJoin<>(props1, INNER, Optional.empty(), formats2, left1, right1, JOIN_EXPRESSION) ) .addEqualityGroup( - new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME, formats2, left1, right1) + new ForeignKeyTableTableJoin<>(props1, INNER, Optional.empty(), formats1, left2, right1, JOIN_EXPRESSION) ) .addEqualityGroup( - new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME, formats1, left2, right1) + new ForeignKeyTableTableJoin<>(props1, INNER, Optional.empty(), formats1, left1, right2, JOIN_EXPRESSION) ) .addEqualityGroup( - new ForeignKeyTableTableJoin<>(props1, INNER, JOIN_COLUMN_NAME, formats1, left1, right2) - ).testEquals(); + new ForeignKeyTableTableJoin<>(props1, INNER, Optional.empty(), formats1, left1, right1, JOIN_EXPRESSION_2) + ) + .testEquals(); } } diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/plan.json new file mode 100644 index 000000000000..3f439c3a70c9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/plan.json @@ -0,0 +1,249 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='DELIMITED', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='right_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n RIGHT_TABLE.R_ID R_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/spec.json new file mode 100644 index 000000000000..b8fc260415e5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/spec.json @@ -0,0 +1,273 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438334984, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should allow (default) key feature mismatch", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : "1", + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : "0", + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : "10", + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : "1", + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : "10", + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : "1", + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : "10", + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : "1", + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', key_format='DELIMITED', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table JOIN right_table ON foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "DELIMITED" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_(default)_key_feature_mismatch/7.1.0_1627438334984/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/plan.json new file mode 100644 index 000000000000..16ee31928186 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/plan.json @@ -0,0 +1,277 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (FORMAT='AVRO', KAFKA_TOPIC='left_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='right_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n RIGHT_TABLE.R_ID R_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "formats" : { + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + } + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/spec.json new file mode 100644 index 000000000000..6a07000a8698 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/spec.json @@ -0,0 +1,396 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438334610, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should allow format mismatch without repartitioning", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "keySchema" : "long", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "FOREIGN_KEY", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "keyFormat" : "AVRO", + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', format='AVRO');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table JOIN right_table ON foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "AVRO" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keySchema" : "long", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "R_ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "F1", + "type" : [ "null", "string" ], + "default" : null + } ] + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "long", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "FOREIGN_KEY", + "type" : [ "null", "long" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.LeftTableKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "keySchema" : "long", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "FOREIGN_KEY", + "type" : [ "null", "long" ], + "default" : null + } ] + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "AVRO", + "properties" : { + "fullSchemaName" : "io.confluent.ksql.avro_schemas.OutputKey" + }, + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "keySchema" : "long", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "R_ID", + "type" : [ "null", "long" ], + "default" : null + }, { + "name" : "NAME", + "type" : [ "null", "string" ], + "default" : null + }, { + "name" : "F1", + "type" : [ "null", "string" ], + "default" : null + } ] + } + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_format_mismatch_without_repartitioning/7.1.0_1627438334610/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/plan.json new file mode 100644 index 000000000000..13abd53dbfb8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/plan.json @@ -0,0 +1,249 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='right_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n RIGHT_TABLE.R_ID R_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/spec.json new file mode 100644 index 000000000000..3e292273bb7b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/spec.json @@ -0,0 +1,273 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438333848, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should allow key format mismatch", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table JOIN right_table ON foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_key_format_mismatch/7.1.0_1627438333848/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/plan.json new file mode 100644 index 000000000000..6f92ad691952 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', PARTITIONS=1, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', PARTITIONS=2, VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n RIGHT_TABLE.R_ID R_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/spec.json new file mode 100644 index 000000000000..dd73813d9bf8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/spec.json @@ -0,0 +1,269 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438334234, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should allow partition count mismatch", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 2 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 1 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON', partitions=1);", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON', partitions=2);", "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table JOIN right_table ON foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 1 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 2 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_partition_count_mismatch/7.1.0_1627438334234/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/plan.json new file mode 100644 index 000000000000..258127969e14 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/spec.json new file mode 100644 index 000000000000..724b6b3a00b2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/spec.json @@ -0,0 +1,265 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438332148, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should allow to omit join columns in projection", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT l_id, name, f1 FROM left_table JOIN right_table ON foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_allow_to_omit_join_columns_in_projection/7.1.0_1627438332148/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/plan.json new file mode 100644 index 000000000000..0ac83b3597aa --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT *\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "LEFT_TABLE_NAME AS LEFT_TABLE_NAME", "LEFT_TABLE_FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "RIGHT_TABLE_R_ID AS RIGHT_TABLE_R_ID", "RIGHT_TABLE_F1 AS RIGHT_TABLE_F1", "RIGHT_TABLE_F2 AS RIGHT_TABLE_F2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/spec.json new file mode 100644 index 000000000000..94fa72eda777 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/spec.json @@ -0,0 +1,213 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438332551, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support `SELECT *`", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "LEFT_TABLE_NAME" : "zero", + "LEFT_TABLE_FOREIGN_KEY" : 0, + "RIGHT_TABLE_R_ID" : 0, + "RIGHT_TABLE_F1" : "blah", + "RIGHT_TABLE_F2" : 4 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT * FROM left_table JOIN right_table ON foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_`SELECT__`/7.1.0_1627438332551/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/plan.json new file mode 100644 index 000000000000..46b36495c340 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n RIGHT_TABLE.R_ID R_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON ((RIGHT_TABLE.R_ID = LEFT_TABLE.FOREIGN_KEY))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/spec.json new file mode 100644 index 000000000000..feb6ef5211d8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/spec.json @@ -0,0 +1,268 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438331306, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support flipped join condition", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table JOIN right_table ON r_id = foreign_key;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition/7.1.0_1627438331306/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/plan.json new file mode 100644 index 000000000000..f52f98434118 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LT.ID LT_ID,\n RT.ID RT_ID_ALIAS,\n LT.NAME NAME_ALIAS,\n RT.F1 F1\nFROM LEFT_TABLE LT\nLEFT OUTER JOIN RIGHT_TABLE RT ON ((RT.ID = LT.FOREIGN_KEY))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`LT_ID` BIGINT KEY, `RT_ID_ALIAS` BIGINT, `NAME_ALIAS` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LT_ID" ], + "selectExpressions" : [ "NAME AS LT_NAME", "FOREIGN_KEY AS LT_FOREIGN_KEY", "ROWTIME AS LT_ROWTIME", "ID AS LT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RT_ID" ], + "selectExpressions" : [ "F1 AS RT_F1", "F2 AS RT_F2", "ROWTIME AS RT_ROWTIME", "ID AS RT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LT_FOREIGN_KEY" + }, + "keyColumnNames" : [ "LT_ID" ], + "selectExpressions" : [ "RT_ID AS RT_ID_ALIAS", "LT_NAME AS NAME_ALIAS", "RT_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/spec.json new file mode 100644 index 000000000000..ecd02750f119 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/spec.json @@ -0,0 +1,321 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438331684, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LT_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LT_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_ID` BIGINT, `RT_F1` STRING, `RT_F2` BIGINT, `RT_ROWTIME` BIGINT, `RT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`LT_ID` BIGINT KEY, `RT_ID_ALIAS` BIGINT, `NAME_ALIAS` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`LT_ID` BIGINT KEY, `RT_ID_ALIAS` BIGINT, `NAME_ALIAS` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RT_ID` BIGINT KEY, `RT_F1` STRING, `RT_F2` BIGINT, `RT_ROWTIME` BIGINT, `RT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support flipped join condition with aliases", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : null, + "timestamp" : 19000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID_ALIAS" : null, + "NAME_ALIAS" : "zero", + "F1" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID_ALIAS" : 0, + "NAME_ALIAS" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID_ALIAS" : 0, + "NAME_ALIAS" : "zero", + "F1" : "blah" + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "RT_ID_ALIAS" : null, + "NAME_ALIAS" : "foo", + "F1" : null + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID_ALIAS" : 0, + "NAME_ALIAS" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "RT_ID_ALIAS" : 0, + "NAME_ALIAS" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID_ALIAS" : null, + "NAME_ALIAS" : "zero", + "F1" : null + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "RT_ID_ALIAS" : null, + "NAME_ALIAS" : "bar", + "F1" : null + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null, + "timestamp" : 19000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT lt.id, rt.id AS rt_id_alias, name AS name_alias, rt.f1 FROM left_table AS lt LEFT JOIN right_table AS rt ON rt.id = lt.foreign_key;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`LT_ID` BIGINT KEY, `RT_ID_ALIAS` BIGINT, `NAME_ALIAS` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_flipped_join_condition_with_aliases/7.1.0_1627438331684/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/plan.json new file mode 100644 index 000000000000..2bb12f2cc029 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/plan.json @@ -0,0 +1,249 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, L_ID_2_FOREIGN_KEY BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='left_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='right_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n LEFT_TABLE.L_ID_2_FOREIGN_KEY L_ID_2_FOREIGN_KEY,\n RIGHT_TABLE.R_ID R_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.L_ID_2_FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID", "LEFT_TABLE_L_ID_2_FOREIGN_KEY" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "VALUE AS LEFT_TABLE_VALUE", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID", "L_ID_2_FOREIGN_KEY AS LEFT_TABLE_L_ID_2_FOREIGN_KEY" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "leftJoinExpression" : "LEFT_TABLE_L_ID_2_FOREIGN_KEY" + }, + "keyColumnNames" : [ "L_ID", "L_ID_2_FOREIGN_KEY" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/spec.json new file mode 100644 index 000000000000..9d7f16d9eab2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/spec.json @@ -0,0 +1,302 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438328984, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_L_ID_2_FOREIGN_KEY` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_VALUE` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `LEFT_TABLE_L_ID_2_FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_L_ID_2_FOREIGN_KEY` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_VALUE` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `LEFT_TABLE_L_ID_2_FOREIGN_KEY` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support inner join with left key-column expression", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : { + "L_ID" : 0, + "L_ID_2_FOREIGN_KEY" : 100 + }, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : { + "L_ID" : 10, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "NAME" : "bar", + "VALUE" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 10, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 10, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, l_id_2_foreign_key BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", "CREATE TABLE output AS SELECT l_id, l_id_2_foreign_key, r_id, name, f1 FROM left_table JOIN right_table ON l_id_2_foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_key-column_expression/7.1.0_1627438328984/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/plan.json new file mode 100644 index 000000000000..b7e97481f1cd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n RIGHT_TABLE.R_ID R_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/spec.json new file mode 100644 index 000000000000..67d835043b82 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/spec.json @@ -0,0 +1,268 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438328037, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support inner join with left value-column expression", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table JOIN right_table ON foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression/7.1.0_1627438328037/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/plan.json new file mode 100644 index 000000000000..2d7ee2b35115 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LT.ID LT_ID,\n RT.ID RT_ID_ALIAS,\n LT.NAME NAME_ALIAS,\n RT.F1 F1\nFROM LEFT_TABLE LT\nINNER JOIN RIGHT_TABLE RT ON ((LT.FOREIGN_KEY = RT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`LT_ID` BIGINT KEY, `RT_ID_ALIAS` BIGINT, `NAME_ALIAS` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LT_ID" ], + "selectExpressions" : [ "NAME AS LT_NAME", "FOREIGN_KEY AS LT_FOREIGN_KEY", "ROWTIME AS LT_ROWTIME", "ID AS LT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RT_ID" ], + "selectExpressions" : [ "F1 AS RT_F1", "F2 AS RT_F2", "ROWTIME AS RT_ROWTIME", "ID AS RT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LT_FOREIGN_KEY" + }, + "keyColumnNames" : [ "LT_ID" ], + "selectExpressions" : [ "RT_ID AS RT_ID_ALIAS", "LT_NAME AS NAME_ALIAS", "RT_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/spec.json new file mode 100644 index 000000000000..5745be4c4533 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/spec.json @@ -0,0 +1,268 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438329904, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LT_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LT_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_ID` BIGINT, `RT_F1` STRING, `RT_F2` BIGINT, `RT_ROWTIME` BIGINT, `RT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`LT_ID` BIGINT KEY, `RT_ID_ALIAS` BIGINT, `NAME_ALIAS` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`LT_ID` BIGINT KEY, `RT_ID_ALIAS` BIGINT, `NAME_ALIAS` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RT_ID` BIGINT KEY, `RT_F1` STRING, `RT_F2` BIGINT, `RT_ROWTIME` BIGINT, `RT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support inner join with left value-column expression - with aliases", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID_ALIAS" : 0, + "NAME_ALIAS" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID_ALIAS" : 0, + "NAME_ALIAS" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "RT_ID_ALIAS" : 0, + "NAME_ALIAS" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT lt.id, rt.id AS rt_id_alias, name AS name_alias, rt.f1 FROM left_table AS lt JOIN right_table AS rt ON lt.foreign_key = rt.id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`LT_ID` BIGINT KEY, `RT_ID_ALIAS` BIGINT, `NAME_ALIAS` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_inner_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438329904/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/plan.json new file mode 100644 index 000000000000..2616b1f14d24 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/plan.json @@ -0,0 +1,249 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, L_ID_2_FOREIGN_KEY BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='left_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='right_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n LEFT_TABLE.L_ID_2_FOREIGN_KEY L_ID_2_FOREIGN_KEY,\n RIGHT_TABLE.R_ID R_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nLEFT OUTER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.L_ID_2_FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID", "LEFT_TABLE_L_ID_2_FOREIGN_KEY" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "VALUE AS LEFT_TABLE_VALUE", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID", "L_ID_2_FOREIGN_KEY AS LEFT_TABLE_L_ID_2_FOREIGN_KEY" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "leftJoinExpression" : "LEFT_TABLE_L_ID_2_FOREIGN_KEY" + }, + "keyColumnNames" : [ "L_ID", "L_ID_2_FOREIGN_KEY" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/spec.json new file mode 100644 index 000000000000..19baff082b5a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/spec.json @@ -0,0 +1,373 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438329411, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_L_ID_2_FOREIGN_KEY` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_VALUE` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `LEFT_TABLE_L_ID_2_FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_L_ID_2_FOREIGN_KEY` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_VALUE` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `LEFT_TABLE_L_ID_2_FOREIGN_KEY` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support left join with left key-column expression", + "inputs" : [ { + "topic" : "left_topic", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : { + "L_ID" : 0, + "L_ID_2_FOREIGN_KEY" : 100 + }, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : { + "L_ID" : 10, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "NAME" : "bar", + "VALUE" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : null, + "timestamp" : 18000 + }, { + "topic" : "left_topic", + "key" : { + "L_ID" : 0, + "L_ID_2_FOREIGN_KEY" : 100 + }, + "value" : null, + "timestamp" : 19000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "R_ID" : null, + "NAME" : "zero", + "F1" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 0, + "L_ID_2_FOREIGN_KEY" : 100 + }, + "value" : { + "R_ID" : null, + "NAME" : "foo", + "F1" : null + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 10, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "R_ID" : null, + "NAME" : "zero", + "F1" : null + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 10, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : { + "R_ID" : null, + "NAME" : "bar", + "F1" : null + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 1, + "L_ID_2_FOREIGN_KEY" : 0 + }, + "value" : null, + "timestamp" : 18000 + }, { + "topic" : "OUTPUT", + "key" : { + "L_ID" : 0, + "L_ID_2_FOREIGN_KEY" : 100 + }, + "value" : null, + "timestamp" : 19000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, l_id_2_foreign_key BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", "CREATE TABLE output AS SELECT l_id, l_id_2_foreign_key, r_id, name, f1 FROM left_table LEFT JOIN right_table ON l_id_2_foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `L_ID_2_FOREIGN_KEY` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_key-column_expression/7.1.0_1627438329411/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/plan.json new file mode 100644 index 000000000000..5279a45eadf2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n RIGHT_TABLE.R_ID R_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nLEFT OUTER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/spec.json new file mode 100644 index 000000000000..60c13f3f8fc5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/spec.json @@ -0,0 +1,321 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438328502, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support left join with left value-column expression", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : null, + "timestamp" : 19000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : null, + "NAME" : "zero", + "F1" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "R_ID" : null, + "NAME" : "foo", + "F1" : null + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : null, + "NAME" : "zero", + "F1" : null + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "R_ID" : null, + "NAME" : "bar", + "F1" : null + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null, + "timestamp" : 19000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table LEFT JOIN right_table ON foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression/7.1.0_1627438328502/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/plan.json new file mode 100644 index 000000000000..36436f9716df --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LT.ID LT_ID,\n RT.ID RT_ID,\n LT.NAME NAME,\n RT.F1 F1\nFROM LEFT_TABLE LT\nLEFT OUTER JOIN RIGHT_TABLE RT ON ((LT.FOREIGN_KEY = RT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`LT_ID` BIGINT KEY, `RT_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LT_ID" ], + "selectExpressions" : [ "NAME AS LT_NAME", "FOREIGN_KEY AS LT_FOREIGN_KEY", "ROWTIME AS LT_ROWTIME", "ID AS LT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RT_ID" ], + "selectExpressions" : [ "F1 AS RT_F1", "F2 AS RT_F2", "ROWTIME AS RT_ROWTIME", "ID AS RT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LT_FOREIGN_KEY" + }, + "keyColumnNames" : [ "LT_ID" ], + "selectExpressions" : [ "RT_ID AS RT_ID", "LT_NAME AS NAME", "RT_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/spec.json new file mode 100644 index 000000000000..fb585cb87ee2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/spec.json @@ -0,0 +1,321 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438330284, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LT_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LT_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_ID` BIGINT, `RT_F1` STRING, `RT_F2` BIGINT, `RT_ROWTIME` BIGINT, `RT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`LT_ID` BIGINT KEY, `RT_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`LT_ID` BIGINT KEY, `RT_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RT_ID` BIGINT KEY, `RT_F1` STRING, `RT_F2` BIGINT, `RT_ROWTIME` BIGINT, `RT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support left join with left value-column expression - with aliases", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : null, + "timestamp" : 19000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID" : null, + "NAME" : "zero", + "F1" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "RT_ID" : null, + "NAME" : "foo", + "F1" : null + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "RT_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RT_ID" : null, + "NAME" : "zero", + "F1" : null + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "RT_ID" : null, + "NAME" : "bar", + "F1" : null + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : null, + "timestamp" : 19000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT lt.id, rt.id, name, rt.f1 FROM left_table AS lt LEFT JOIN right_table AS rt ON lt.foreign_key = rt.id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`LT_ID` BIGINT KEY, `RT_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_left_join_with_left_value-column_expression_-_with_aliases/7.1.0_1627438330284/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/plan.json new file mode 100644 index 000000000000..8dd323135622 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n RIGHT_TABLE.R_ID R_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON (((LEFT_TABLE.FOREIGN_KEY + 1) = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "(LEFT_TABLE_FOREIGN_KEY + 1)" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/spec.json new file mode 100644 index 000000000000..389ed20e9e84 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/spec.json @@ -0,0 +1,268 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438330922, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support non-column reference in left join expression", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : -1 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 99 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : -1 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table JOIN right_table ON foreign_key + 1 = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression/7.1.0_1627438330922/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/plan.json new file mode 100644 index 000000000000..b0404346a9e0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LT.L_ID L_ID,\n RIGHT_TABLE.R_ID R_ID,\n LT.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LT\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON (((LT.FOREIGN_KEY + 1) = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LT_L_ID" ], + "selectExpressions" : [ "NAME AS LT_NAME", "FOREIGN_KEY AS LT_FOREIGN_KEY", "ROWTIME AS LT_ROWTIME", "L_ID AS LT_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "(LT_FOREIGN_KEY + 1)" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LT_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/spec.json new file mode 100644 index 000000000000..45b7341d3a20 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/spec.json @@ -0,0 +1,268 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627505730914, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LT_L_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LT_L_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support non-column reference in left join expression with alias", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : -1 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 99 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : -1 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT lt.l_id, r_id, lt.name, f1 FROM left_table AS lt JOIN right_table ON lt.foreign_key + 1 = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_alias/7.1.0_1627505730914/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/plan.json new file mode 100644 index 000000000000..8dd323135622 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n RIGHT_TABLE.R_ID R_ID,\n LEFT_TABLE.NAME NAME,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON (((LEFT_TABLE.FOREIGN_KEY + 1) = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "(LEFT_TABLE_FOREIGN_KEY + 1)" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS R_ID", "LEFT_TABLE_NAME AS NAME", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/spec.json new file mode 100644 index 000000000000..5b07d34d6590 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/spec.json @@ -0,0 +1,268 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627505730530, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support non-column reference in left join expression with qualifier", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : -1 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "FOREIGN_KEY" : 99 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "bar", + "FOREIGN_KEY" : -1 + }, + "timestamp" : 16000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "R_ID" : 0, + "NAME" : "zero", + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "R_ID" : 0, + "NAME" : "bar", + "F1" : "a" + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : null, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table JOIN right_table ON left_table.foreign_key + 1 = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `R_ID` BIGINT, `NAME` STRING, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_non-column_reference_in_left_join_expression_with_qualifier/7.1.0_1627505730530/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/plan.json new file mode 100644 index 000000000000..d58177f7e25f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LT.*,\n RT.*,\n LT.NAME NAME_ALIAS\nFROM LEFT_TABLE LT\nLEFT OUTER JOIN RIGHT_TABLE RT ON ((LT.FOREIGN_KEY = RT.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`LT_L_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `RT_R_ID` BIGINT, `RT_F1` STRING, `RT_F2` BIGINT, `NAME_ALIAS` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LT_L_ID" ], + "selectExpressions" : [ "NAME AS LT_NAME", "FOREIGN_KEY AS LT_FOREIGN_KEY", "ROWTIME AS LT_ROWTIME", "L_ID AS LT_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RT_R_ID" ], + "selectExpressions" : [ "F1 AS RT_F1", "F2 AS RT_F2", "ROWTIME AS RT_ROWTIME", "R_ID AS RT_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LT_FOREIGN_KEY" + }, + "keyColumnNames" : [ "LT_L_ID" ], + "selectExpressions" : [ "LT_NAME AS LT_NAME", "LT_FOREIGN_KEY AS LT_FOREIGN_KEY", "RT_R_ID AS RT_R_ID", "RT_F1 AS RT_F1", "RT_F2 AS RT_F2", "LT_NAME AS NAME_ALIAS" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/spec.json new file mode 100644 index 000000000000..99d372e720c0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/spec.json @@ -0,0 +1,245 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438333496, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LT_L_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LT_L_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_L_ID` BIGINT, `RT_F1` STRING, `RT_F2` BIGINT, `RT_ROWTIME` BIGINT, `RT_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`LT_L_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `RT_R_ID` BIGINT, `RT_F1` STRING, `RT_F2` BIGINT, `NAME_ALIAS` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`LT_L_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `RT_R_ID` BIGINT, `RT_F1` STRING, `RT_F2` BIGINT, `NAME_ALIAS` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RT_R_ID` BIGINT KEY, `RT_F1` STRING, `RT_F2` BIGINT, `RT_ROWTIME` BIGINT, `RT_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support qualified `SELECT *` for both input using aliases", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "LT_NAME" : "zero", + "LT_FOREIGN_KEY" : 0, + "RT_R_ID" : null, + "RT_F1" : null, + "RT_F2" : null, + "NAME_ALIAS" : "zero" + } + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "LT_NAME" : "zero", + "LT_FOREIGN_KEY" : 0, + "RT_R_ID" : 0, + "RT_F1" : "blah", + "RT_F2" : 4, + "NAME_ALIAS" : "zero" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "LT_NAME" : "zero", + "LT_FOREIGN_KEY" : 0, + "RT_R_ID" : 0, + "RT_F1" : "blah", + "RT_F2" : 4, + "NAME_ALIAS" : "zero" + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT lt.*, rt.*, lt.name AS name_alias FROM left_table AS lt LEFT JOIN right_table AS rt ON foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`LT_L_ID` BIGINT KEY, `LT_NAME` STRING, `LT_FOREIGN_KEY` BIGINT, `RT_R_ID` BIGINT, `RT_F1` STRING, `RT_F2` BIGINT, `NAME_ALIAS` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_both_input_using_aliases/7.1.0_1627438333496/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/plan.json new file mode 100644 index 000000000000..5efe6fd9a402 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.*,\n RIGHT_TABLE.F1 F1\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `F1` STRING", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "LEFT_TABLE_NAME AS LEFT_TABLE_NAME", "LEFT_TABLE_FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "RIGHT_TABLE_F1 AS F1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/spec.json new file mode 100644 index 000000000000..694c04a86449 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/spec.json @@ -0,0 +1,211 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438333184, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support qualified `SELECT *` for left input", + "inputs" : [ { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : null, + "timestamp" : 17000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "LEFT_TABLE_NAME" : "zero", + "LEFT_TABLE_FOREIGN_KEY" : 0, + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 17000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT left_table.*, right_table.f1 FROM left_table JOIN right_table ON foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_left_input/7.1.0_1627438333184/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/plan.json new file mode 100644 index 000000000000..e6295ec05c51 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, NAME STRING, FOREIGN_KEY BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n RIGHT_TABLE.*\nFROM LEFT_TABLE LEFT_TABLE\nLEFT OUTER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "NAME AS LEFT_TABLE_NAME", "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F1 AS RIGHT_TABLE_F1", "F2 AS RIGHT_TABLE_F2", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "RIGHT_TABLE_R_ID AS RIGHT_TABLE_R_ID", "RIGHT_TABLE_F1 AS RIGHT_TABLE_F1", "RIGHT_TABLE_F2 AS RIGHT_TABLE_F2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/spec.json new file mode 100644 index 000000000000..52c9301a18df --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/spec.json @@ -0,0 +1,237 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438332834, + "path" : "query-validation-tests/fk-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_NAME` STRING, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should support qualified `SELECT *` for right input", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 4 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "FOREIGN_KEY" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RIGHT_TABLE_R_ID" : null, + "RIGHT_TABLE_F1" : null, + "RIGHT_TABLE_F2" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RIGHT_TABLE_R_ID" : 0, + "RIGHT_TABLE_F1" : "blah", + "RIGHT_TABLE_F2" : 4 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "RIGHT_TABLE_R_ID" : 0, + "RIGHT_TABLE_F1" : "blah", + "RIGHT_TABLE_F2" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", "CREATE TABLE output AS SELECT l_id, right_table.* FROM left_table LEFT JOIN right_table ON foreign_key = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `NAME` STRING, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F1` STRING, `RIGHT_TABLE_F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/topology new file mode 100644 index 000000000000..0d55155a9d7f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-join_-_Should_support_qualified_`SELECT__`_for_right_input/7.1.0_1627438332834/topology @@ -0,0 +1,67 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000026 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000026 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000027 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasLeft + Processor: KTABLE-TOSTREAM-0000000027 (stores: []) + --> KSTREAM-SINK-0000000028 + <-- KTABLE-TRANSFORMVALUES-0000000026 + Sink: KSTREAM-SINK-0000000028 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000027 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasRight (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasRight + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/plan.json new file mode 100644 index 000000000000..25531e302bd4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/plan.json @@ -0,0 +1,318 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, FOREIGN_KEY BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='left_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE MIDDLE_TABLE (M_ID BIGINT PRIMARY KEY, F2 BIGINT, OTHER STRING) WITH (FORMAT='JSON', KAFKA_TOPIC='middle_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "MIDDLE_TABLE", + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F3 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='right_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n MIDDLE_TABLE.M_ID M_ID,\n LEFT_TABLE.FOREIGN_KEY FOREIGN_KEY,\n MIDDLE_TABLE.F2 F2,\n RIGHT_TABLE.F3 F3\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN MIDDLE_TABLE MIDDLE_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = MIDDLE_TABLE.M_ID))\nLEFT OUTER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.L_ID = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `M_ID` BIGINT, `FOREIGN_KEY` BIGINT, `F2` BIGINT, `F3` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "MIDDLE_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "MIDDLE_TABLE_M_ID" ], + "selectExpressions" : [ "F2 AS MIDDLE_TABLE_F2", "OTHER AS MIDDLE_TABLE_OTHER", "ROWTIME AS MIDDLE_TABLE_ROWTIME", "M_ID AS MIDDLE_TABLE_M_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F3 AS RIGHT_TABLE_F3", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "keyColName" : "LEFT_TABLE_L_ID" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "MIDDLE_TABLE_M_ID AS M_ID", "LEFT_TABLE_FOREIGN_KEY AS FOREIGN_KEY", "MIDDLE_TABLE_F2 AS F2", "RIGHT_TABLE_F3 AS F3" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/spec.json new file mode 100644 index 000000000000..1692c1e0d2d2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/spec.json @@ -0,0 +1,328 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438335608, + "path" : "query-validation-tests/fk-n-way-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasL_Left" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasL_Right" : { + "schema" : "`MIDDLE_TABLE_M_ID` BIGINT KEY, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `MIDDLE_TABLE_ROWTIME` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Right.Source" : { + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `M_ID` BIGINT, `FOREIGN_KEY` BIGINT, `F2` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `M_ID` BIGINT, `FOREIGN_KEY` BIGINT, `F2` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F3` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.L_Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `MIDDLE_TABLE_ROWTIME` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should allow fk join at start of n-way join", + "inputs" : [ { + "topic" : "middle_topic", + "key" : 0, + "value" : { + "F2" : 100, + "OTHER" : "unused" + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "middle_topic", + "key" : 8, + "value" : { + "F2" : 10, + "OTHER" : "unused" + }, + "timestamp" : 13000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "FOREIGN_KEY" : 8 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "M_ID" : 0, + "FOREIGN_KEY" : 0, + "F2" : 100, + "F3" : null + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "M_ID" : 0, + "FOREIGN_KEY" : 0, + "F2" : 100, + "F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "M_ID" : 8, + "FOREIGN_KEY" : 8, + "F2" : 10, + "F3" : 4 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "middle_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, foreign_key BIGINT) WITH (kafka_topic='left_topic', format='JSON');", "CREATE TABLE middle_table (m_id BIGINT PRIMARY KEY, f2 BIGINT, other STRING) WITH (kafka_topic='middle_topic', format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f3 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", "CREATE TABLE output AS SELECT l_id, m_id, foreign_key, f2, f3 FROM left_table JOIN middle_table ON foreign_key = m_id LEFT JOIN right_table ON l_id = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "MIDDLE_TABLE", + "type" : "TABLE", + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `M_ID` BIGINT, `FOREIGN_KEY` BIGINT, `F2` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Left-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "middle_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/topology new file mode 100644 index 000000000000..629223791785 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join/7.1.0_1627438335608/topology @@ -0,0 +1,90 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000027 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000028 + Processor: KTABLE-SOURCE-0000000028 (stores: []) + --> KTABLE-MAPVALUES-0000000029 + <-- KSTREAM-SOURCE-0000000027 + Processor: KTABLE-MAPVALUES-0000000029 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000030 + <-- KTABLE-SOURCE-0000000028 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-TRANSFORMVALUES-0000000030 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000029 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-JOINTHIS-0000000033 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000034 + <-- KTABLE-TRANSFORMVALUES-0000000030 + Processor: KTABLE-JOINOTHER-0000000034 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-MERGE-0000000032 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000033 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000032 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MERGE-0000000032 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000035 + <-- KTABLE-JOINTHIS-0000000033, KTABLE-JOINOTHER-0000000034 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasL_Left + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000035 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000036 + <-- KTABLE-MERGE-0000000032 + Processor: PrependAliasL_Left (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasL_Left + Processor: KTABLE-TOSTREAM-0000000036 (stores: []) + --> KSTREAM-SINK-0000000037 + <-- KTABLE-TRANSFORMVALUES-0000000035 + Sink: KSTREAM-SINK-0000000037 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000036 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [middle_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasL_Right (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasL_Right + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/plan.json new file mode 100644 index 000000000000..029f1410f932 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/plan.json @@ -0,0 +1,318 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, FOREIGN_KEY BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='left_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE MIDDLE_TABLE (M_ID BIGINT PRIMARY KEY, F2 BIGINT, OTHER STRING) WITH (FORMAT='JSON', KAFKA_TOPIC='middle_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "MIDDLE_TABLE", + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F3 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='right_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n RIGHT_TABLE.R_ID R_ID,\n MIDDLE_TABLE.M_ID M_ID,\n LEFT_TABLE.FOREIGN_KEY FOREIGN_KEY,\n MIDDLE_TABLE.F2 F2,\n RIGHT_TABLE.F3 F3\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN MIDDLE_TABLE MIDDLE_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = MIDDLE_TABLE.M_ID))\nLEFT OUTER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.L_ID = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`R_ID` BIGINT KEY, `M_ID` BIGINT, `FOREIGN_KEY` BIGINT, `F2` BIGINT, `F3` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "MIDDLE_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "MIDDLE_TABLE_M_ID" ], + "selectExpressions" : [ "F2 AS MIDDLE_TABLE_F2", "OTHER AS MIDDLE_TABLE_OTHER", "ROWTIME AS MIDDLE_TABLE_ROWTIME", "M_ID AS MIDDLE_TABLE_M_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F3 AS RIGHT_TABLE_F3", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "keyColName" : "RIGHT_TABLE_R_ID" + }, + "keyColumnNames" : [ "R_ID" ], + "selectExpressions" : [ "MIDDLE_TABLE_M_ID AS M_ID", "LEFT_TABLE_FOREIGN_KEY AS FOREIGN_KEY", "MIDDLE_TABLE_F2 AS F2", "RIGHT_TABLE_F3 AS F3" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/spec.json new file mode 100644 index 000000000000..cb963da85c34 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/spec.json @@ -0,0 +1,328 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438336446, + "path" : "query-validation-tests/fk-n-way-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasL_Left" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasL_Right" : { + "schema" : "`MIDDLE_TABLE_M_ID` BIGINT KEY, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `MIDDLE_TABLE_ROWTIME` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Right.Source" : { + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`R_ID` BIGINT KEY, `M_ID` BIGINT, `FOREIGN_KEY` BIGINT, `F2` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`R_ID` BIGINT KEY, `M_ID` BIGINT, `FOREIGN_KEY` BIGINT, `F2` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F3` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.L_Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `MIDDLE_TABLE_ROWTIME` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should allow fk join at start of n-way join - alternative key expression in projection", + "inputs" : [ { + "topic" : "middle_topic", + "key" : 0, + "value" : { + "F2" : 100, + "OTHER" : "unused" + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "middle_topic", + "key" : 8, + "value" : { + "F2" : 10, + "OTHER" : "unused" + }, + "timestamp" : 13000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "FOREIGN_KEY" : 8 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "M_ID" : 0, + "FOREIGN_KEY" : 0, + "F2" : 100, + "F3" : null + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "M_ID" : 0, + "FOREIGN_KEY" : 0, + "F2" : 100, + "F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "M_ID" : 8, + "FOREIGN_KEY" : 8, + "F2" : 10, + "F3" : 4 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "middle_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, foreign_key BIGINT) WITH (kafka_topic='left_topic', format='JSON');", "CREATE TABLE middle_table (m_id BIGINT PRIMARY KEY, f2 BIGINT, other STRING) WITH (kafka_topic='middle_topic', format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f3 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", "CREATE TABLE output AS SELECT r_id, m_id, foreign_key, f2, f3 FROM left_table JOIN middle_table ON foreign_key = m_id LEFT JOIN right_table ON l_id = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "MIDDLE_TABLE", + "type" : "TABLE", + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `M_ID` BIGINT, `FOREIGN_KEY` BIGINT, `F2` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Left-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "middle_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/topology new file mode 100644 index 000000000000..629223791785 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_alternative_key_expression_in_projection/7.1.0_1627438336446/topology @@ -0,0 +1,90 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000027 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000028 + Processor: KTABLE-SOURCE-0000000028 (stores: []) + --> KTABLE-MAPVALUES-0000000029 + <-- KSTREAM-SOURCE-0000000027 + Processor: KTABLE-MAPVALUES-0000000029 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000030 + <-- KTABLE-SOURCE-0000000028 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-TRANSFORMVALUES-0000000030 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000029 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-JOINTHIS-0000000033 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000034 + <-- KTABLE-TRANSFORMVALUES-0000000030 + Processor: KTABLE-JOINOTHER-0000000034 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-MERGE-0000000032 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000033 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000032 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MERGE-0000000032 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000035 + <-- KTABLE-JOINTHIS-0000000033, KTABLE-JOINOTHER-0000000034 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasL_Left + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000035 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000036 + <-- KTABLE-MERGE-0000000032 + Processor: PrependAliasL_Left (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasL_Left + Processor: KTABLE-TOSTREAM-0000000036 (stores: []) + --> KSTREAM-SINK-0000000037 + <-- KTABLE-TRANSFORMVALUES-0000000035 + Sink: KSTREAM-SINK-0000000037 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000036 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [middle_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasL_Right (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasL_Right + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/plan.json new file mode 100644 index 000000000000..ef1fd6fbc806 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/plan.json @@ -0,0 +1,318 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (ID1 BIGINT PRIMARY KEY, F1 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='left_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`ID1` BIGINT KEY, `F1` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE MIDDLE_TABLE (ID2 BIGINT PRIMARY KEY, F2 BIGINT, OTHER STRING) WITH (FORMAT='JSON', KAFKA_TOPIC='middle_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "MIDDLE_TABLE", + "schema" : "`ID2` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (ID3 BIGINT PRIMARY KEY, F3 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='right_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`ID3` BIGINT KEY, `F3` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.ID1 ID1,\n MIDDLE_TABLE.*,\n RIGHT_TABLE.*\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN MIDDLE_TABLE MIDDLE_TABLE ON ((LEFT_TABLE.F1 = MIDDLE_TABLE.ID2))\nLEFT OUTER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.ID1 = RIGHT_TABLE.ID3))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`ID1` BIGINT KEY, `MIDDLE_TABLE_ID2` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `RIGHT_TABLE_ID3` BIGINT, `RIGHT_TABLE_F3` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "MIDDLE_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID1` BIGINT KEY, `F1` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_ID1" ], + "selectExpressions" : [ "F1 AS LEFT_TABLE_F1", "ROWTIME AS LEFT_TABLE_ROWTIME", "ID1 AS LEFT_TABLE_ID1" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID2` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "MIDDLE_TABLE_ID2" ], + "selectExpressions" : [ "F2 AS MIDDLE_TABLE_F2", "OTHER AS MIDDLE_TABLE_OTHER", "ROWTIME AS MIDDLE_TABLE_ROWTIME", "ID2 AS MIDDLE_TABLE_ID2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "leftJoinExpression" : "LEFT_TABLE_F1" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`ID3` BIGINT KEY, `F3` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_ID3" ], + "selectExpressions" : [ "F3 AS RIGHT_TABLE_F3", "ROWTIME AS RIGHT_TABLE_ROWTIME", "ID3 AS RIGHT_TABLE_ID3" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "keyColName" : "LEFT_TABLE_ID1" + }, + "keyColumnNames" : [ "ID1" ], + "selectExpressions" : [ "MIDDLE_TABLE_ID2 AS MIDDLE_TABLE_ID2", "MIDDLE_TABLE_F2 AS MIDDLE_TABLE_F2", "MIDDLE_TABLE_OTHER AS MIDDLE_TABLE_OTHER", "RIGHT_TABLE_ID3 AS RIGHT_TABLE_ID3", "RIGHT_TABLE_F3 AS RIGHT_TABLE_F3" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/spec.json new file mode 100644 index 000000000000..64f61ce6cf7e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/spec.json @@ -0,0 +1,331 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438337258, + "path" : "query-validation-tests/fk-n-way-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasL_Left" : { + "schema" : "`LEFT_TABLE_ID1` BIGINT KEY, `LEFT_TABLE_F1` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_ID1` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasL_Right" : { + "schema" : "`MIDDLE_TABLE_ID2` BIGINT KEY, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `MIDDLE_TABLE_ROWTIME` BIGINT, `MIDDLE_TABLE_ID2` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Right.Source" : { + "schema" : "`ID2` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Left.Source" : { + "schema" : "`ID1` BIGINT KEY, `F1` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`ID1` BIGINT KEY, `MIDDLE_TABLE_ID2` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `RIGHT_TABLE_ID3` BIGINT, `RIGHT_TABLE_F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID3` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ID1` BIGINT KEY, `MIDDLE_TABLE_ID2` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `RIGHT_TABLE_ID3` BIGINT, `RIGHT_TABLE_F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_ID3` BIGINT KEY, `RIGHT_TABLE_F3` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_ID3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.L_Join" : { + "schema" : "`LEFT_TABLE_ID1` BIGINT KEY, `LEFT_TABLE_F1` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_ID1` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `MIDDLE_TABLE_ROWTIME` BIGINT, `MIDDLE_TABLE_ID2` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should allow fk join at start of n-way join - qualified select *", + "inputs" : [ { + "topic" : "middle_topic", + "key" : 0, + "value" : { + "F2" : 100, + "OTHER" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "F1" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "middle_topic", + "key" : 8, + "value" : { + "F2" : 10, + "OTHER" : "bar" + }, + "timestamp" : 13000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "F1" : 8 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "MIDDLE_TABLE_ID2" : 0, + "MIDDLE_TABLE_F2" : 100, + "MIDDLE_TABLE_OTHER" : "foo", + "RIGHT_TABLE_ID3" : null, + "RIGHT_TABLE_F3" : null + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "MIDDLE_TABLE_ID2" : 0, + "MIDDLE_TABLE_F2" : 100, + "MIDDLE_TABLE_OTHER" : "foo", + "RIGHT_TABLE_ID3" : 1, + "RIGHT_TABLE_F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "MIDDLE_TABLE_ID2" : 8, + "MIDDLE_TABLE_F2" : 10, + "MIDDLE_TABLE_OTHER" : "bar", + "RIGHT_TABLE_ID3" : 1, + "RIGHT_TABLE_F3" : 4 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "middle_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (id1 BIGINT PRIMARY KEY, f1 BIGINT) WITH (kafka_topic='left_topic', format='JSON');", "CREATE TABLE middle_table (id2 BIGINT PRIMARY KEY, f2 BIGINT, other STRING) WITH (kafka_topic='middle_topic', format='JSON');", "CREATE TABLE right_table (id3 BIGINT PRIMARY KEY, f3 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", "CREATE TABLE output AS SELECT id1, middle_table.*, right_table.* FROM left_table JOIN middle_table ON f1 = id2 LEFT JOIN right_table ON id1 = id3;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`ID1` BIGINT KEY, `F1` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "MIDDLE_TABLE", + "type" : "TABLE", + "schema" : "`ID2` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`ID1` BIGINT KEY, `MIDDLE_TABLE_ID2` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `RIGHT_TABLE_ID3` BIGINT, `RIGHT_TABLE_F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`ID3` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Left-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "middle_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/topology new file mode 100644 index 000000000000..629223791785 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_qualified_select__/7.1.0_1627438337258/topology @@ -0,0 +1,90 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000027 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000028 + Processor: KTABLE-SOURCE-0000000028 (stores: []) + --> KTABLE-MAPVALUES-0000000029 + <-- KSTREAM-SOURCE-0000000027 + Processor: KTABLE-MAPVALUES-0000000029 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000030 + <-- KTABLE-SOURCE-0000000028 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-TRANSFORMVALUES-0000000030 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000029 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-JOINTHIS-0000000033 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000034 + <-- KTABLE-TRANSFORMVALUES-0000000030 + Processor: KTABLE-JOINOTHER-0000000034 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-MERGE-0000000032 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000033 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000032 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MERGE-0000000032 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000035 + <-- KTABLE-JOINTHIS-0000000033, KTABLE-JOINOTHER-0000000034 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasL_Left + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000035 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000036 + <-- KTABLE-MERGE-0000000032 + Processor: PrependAliasL_Left (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasL_Left + Processor: KTABLE-TOSTREAM-0000000036 (stores: []) + --> KSTREAM-SINK-0000000037 + <-- KTABLE-TRANSFORMVALUES-0000000035 + Sink: KSTREAM-SINK-0000000037 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000036 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [middle_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasL_Right (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasL_Right + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/plan.json new file mode 100644 index 000000000000..79878c8e7342 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/plan.json @@ -0,0 +1,318 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, FOREIGN_KEY BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='left_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE MIDDLE_TABLE (M_ID BIGINT PRIMARY KEY, F2 BIGINT, OTHER STRING) WITH (FORMAT='JSON', KAFKA_TOPIC='middle_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "MIDDLE_TABLE", + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F3 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='right_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT *\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN MIDDLE_TABLE MIDDLE_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = MIDDLE_TABLE.M_ID))\nLEFT OUTER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.L_ID = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F3` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "MIDDLE_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "MIDDLE_TABLE_M_ID" ], + "selectExpressions" : [ "F2 AS MIDDLE_TABLE_F2", "OTHER AS MIDDLE_TABLE_OTHER", "ROWTIME AS MIDDLE_TABLE_ROWTIME", "M_ID AS MIDDLE_TABLE_M_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F3 AS RIGHT_TABLE_F3", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "keyColName" : "LEFT_TABLE_L_ID" + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "LEFT_TABLE_FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "MIDDLE_TABLE_M_ID AS MIDDLE_TABLE_M_ID", "MIDDLE_TABLE_F2 AS MIDDLE_TABLE_F2", "MIDDLE_TABLE_OTHER AS MIDDLE_TABLE_OTHER", "RIGHT_TABLE_R_ID AS RIGHT_TABLE_R_ID", "RIGHT_TABLE_F3 AS RIGHT_TABLE_F3" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/spec.json new file mode 100644 index 000000000000..0df2ddc29e16 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/spec.json @@ -0,0 +1,334 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438336855, + "path" : "query-validation-tests/fk-n-way-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasL_Left" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasL_Right" : { + "schema" : "`MIDDLE_TABLE_M_ID` BIGINT KEY, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `MIDDLE_TABLE_ROWTIME` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Right.Source" : { + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F3` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.L_Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `MIDDLE_TABLE_ROWTIME` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should allow fk join at start of n-way join - select *", + "inputs" : [ { + "topic" : "middle_topic", + "key" : 0, + "value" : { + "F2" : 100, + "OTHER" : "unused" + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "middle_topic", + "key" : 8, + "value" : { + "F2" : 10, + "OTHER" : "unused" + }, + "timestamp" : 13000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "FOREIGN_KEY" : 8 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "LEFT_TABLE_FOREIGN_KEY" : 0, + "MIDDLE_TABLE_M_ID" : 0, + "MIDDLE_TABLE_F2" : 100, + "MIDDLE_TABLE_OTHER" : "unused", + "RIGHT_TABLE_R_ID" : null, + "RIGHT_TABLE_F3" : null + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "LEFT_TABLE_FOREIGN_KEY" : 0, + "MIDDLE_TABLE_M_ID" : 0, + "MIDDLE_TABLE_F2" : 100, + "MIDDLE_TABLE_OTHER" : "unused", + "RIGHT_TABLE_R_ID" : 1, + "RIGHT_TABLE_F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "LEFT_TABLE_FOREIGN_KEY" : 8, + "MIDDLE_TABLE_M_ID" : 8, + "MIDDLE_TABLE_F2" : 10, + "MIDDLE_TABLE_OTHER" : "unused", + "RIGHT_TABLE_R_ID" : 1, + "RIGHT_TABLE_F3" : 4 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "middle_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, foreign_key BIGINT) WITH (kafka_topic='left_topic', format='JSON');", "CREATE TABLE middle_table (m_id BIGINT PRIMARY KEY, f2 BIGINT, other STRING) WITH (kafka_topic='middle_topic', format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f3 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", "CREATE TABLE output AS SELECT * FROM left_table JOIN middle_table ON foreign_key = m_id LEFT JOIN right_table ON l_id = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "MIDDLE_TABLE", + "type" : "TABLE", + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `RIGHT_TABLE_R_ID` BIGINT, `RIGHT_TABLE_F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Left-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "middle_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/topology new file mode 100644 index 000000000000..629223791785 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_select__/7.1.0_1627438336855/topology @@ -0,0 +1,90 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000027 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000028 + Processor: KTABLE-SOURCE-0000000028 (stores: []) + --> KTABLE-MAPVALUES-0000000029 + <-- KSTREAM-SOURCE-0000000027 + Processor: KTABLE-MAPVALUES-0000000029 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000030 + <-- KTABLE-SOURCE-0000000028 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-TRANSFORMVALUES-0000000030 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000029 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-JOINTHIS-0000000033 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000034 + <-- KTABLE-TRANSFORMVALUES-0000000030 + Processor: KTABLE-JOINOTHER-0000000034 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-MERGE-0000000032 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000033 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000032 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MERGE-0000000032 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000035 + <-- KTABLE-JOINTHIS-0000000033, KTABLE-JOINOTHER-0000000034 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasL_Left + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000035 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000036 + <-- KTABLE-MERGE-0000000032 + Processor: PrependAliasL_Left (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasL_Left + Processor: KTABLE-TOSTREAM-0000000036 (stores: []) + --> KSTREAM-SINK-0000000037 + <-- KTABLE-TRANSFORMVALUES-0000000035 + Sink: KSTREAM-SINK-0000000037 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000036 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [middle_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasL_Right (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasL_Right + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/plan.json new file mode 100644 index 000000000000..30611a40721c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/plan.json @@ -0,0 +1,318 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, FOREIGN_KEY BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='left_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE MIDDLE_TABLE (M_ID BIGINT PRIMARY KEY, F2 BIGINT, OTHER STRING) WITH (FORMAT='JSON', KAFKA_TOPIC='middle_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "MIDDLE_TABLE", + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F3 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='right_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LT.L_ID L_ID,\n MT.M_ID M_ID_ALIAS,\n LT.FOREIGN_KEY FOREIGN_KEY,\n MT.F2 MT_F2_ALIAS,\n RT.F3 F3\nFROM LEFT_TABLE LT\nINNER JOIN MIDDLE_TABLE MT ON ((LT.FOREIGN_KEY = MT.M_ID))\nLEFT OUTER JOIN RIGHT_TABLE RT ON ((LT.L_ID = RT.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `M_ID_ALIAS` BIGINT, `FOREIGN_KEY` BIGINT, `MT_F2_ALIAS` BIGINT, `F3` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "MIDDLE_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LT_L_ID" ], + "selectExpressions" : [ "FOREIGN_KEY AS LT_FOREIGN_KEY", "ROWTIME AS LT_ROWTIME", "L_ID AS LT_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "MT_M_ID" ], + "selectExpressions" : [ "F2 AS MT_F2", "OTHER AS MT_OTHER", "ROWTIME AS MT_ROWTIME", "M_ID AS MT_M_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "leftJoinExpression" : "LT_FOREIGN_KEY" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RT_R_ID" ], + "selectExpressions" : [ "F3 AS RT_F3", "ROWTIME AS RT_ROWTIME", "R_ID AS RT_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "keyColName" : "LT_L_ID" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "MT_M_ID AS M_ID_ALIAS", "LT_FOREIGN_KEY AS FOREIGN_KEY", "MT_F2 AS MT_F2_ALIAS", "RT_F3 AS F3" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/spec.json new file mode 100644 index 000000000000..d680e9a06da1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/spec.json @@ -0,0 +1,328 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438337654, + "path" : "query-validation-tests/fk-n-way-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasL_Left" : { + "schema" : "`LT_L_ID` BIGINT KEY, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_L_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasL_Right" : { + "schema" : "`MT_M_ID` BIGINT KEY, `MT_F2` BIGINT, `MT_OTHER` STRING, `MT_ROWTIME` BIGINT, `MT_M_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Right.Source" : { + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `M_ID_ALIAS` BIGINT, `FOREIGN_KEY` BIGINT, `MT_F2_ALIAS` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `M_ID_ALIAS` BIGINT, `FOREIGN_KEY` BIGINT, `MT_F2_ALIAS` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RT_R_ID` BIGINT KEY, `RT_F3` BIGINT, `RT_ROWTIME` BIGINT, `RT_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.L_Join" : { + "schema" : "`LT_L_ID` BIGINT KEY, `LT_FOREIGN_KEY` BIGINT, `LT_ROWTIME` BIGINT, `LT_L_ID` BIGINT, `MT_F2` BIGINT, `MT_OTHER` STRING, `MT_ROWTIME` BIGINT, `MT_M_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should allow fk join at start of n-way join - with aliases", + "inputs" : [ { + "topic" : "middle_topic", + "key" : 0, + "value" : { + "F2" : 100, + "OTHER" : "unused" + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "middle_topic", + "key" : 8, + "value" : { + "F2" : 10, + "OTHER" : "unused" + }, + "timestamp" : 13000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "FOREIGN_KEY" : 8 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "M_ID_ALIAS" : 0, + "FOREIGN_KEY" : 0, + "MT_F2_ALIAS" : 100, + "F3" : null + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "M_ID_ALIAS" : 0, + "FOREIGN_KEY" : 0, + "MT_F2_ALIAS" : 100, + "F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "M_ID_ALIAS" : 8, + "FOREIGN_KEY" : 8, + "MT_F2_ALIAS" : 10, + "F3" : 4 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "middle_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, foreign_key BIGINT) WITH (kafka_topic='left_topic', format='JSON');", "CREATE TABLE middle_table (m_id BIGINT PRIMARY KEY, f2 BIGINT, other STRING) WITH (kafka_topic='middle_topic', format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f3 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", "CREATE TABLE output AS SELECT l_id, m_id AS m_id_alias, foreign_key, mt.f2 AS mt_f2_alias, rt.f3 FROM left_table AS lt JOIN middle_table AS mt ON foreign_key = mt.m_id LEFT JOIN right_table AS rt ON lt.l_id = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "MIDDLE_TABLE", + "type" : "TABLE", + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `M_ID_ALIAS` BIGINT, `FOREIGN_KEY` BIGINT, `MT_F2_ALIAS` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Left-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "middle_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/topology new file mode 100644 index 000000000000..629223791785 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_with_aliases/7.1.0_1627438337654/topology @@ -0,0 +1,90 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000027 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000028 + Processor: KTABLE-SOURCE-0000000028 (stores: []) + --> KTABLE-MAPVALUES-0000000029 + <-- KSTREAM-SOURCE-0000000027 + Processor: KTABLE-MAPVALUES-0000000029 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000030 + <-- KTABLE-SOURCE-0000000028 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-TRANSFORMVALUES-0000000030 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000029 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-JOINTHIS-0000000033 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000034 + <-- KTABLE-TRANSFORMVALUES-0000000030 + Processor: KTABLE-JOINOTHER-0000000034 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-MERGE-0000000032 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000033 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000032 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MERGE-0000000032 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000035 + <-- KTABLE-JOINTHIS-0000000033, KTABLE-JOINOTHER-0000000034 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasL_Left + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000035 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000036 + <-- KTABLE-MERGE-0000000032 + Processor: PrependAliasL_Left (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasL_Left + Processor: KTABLE-TOSTREAM-0000000036 (stores: []) + --> KSTREAM-SINK-0000000037 + <-- KTABLE-TRANSFORMVALUES-0000000035 + Sink: KSTREAM-SINK-0000000037 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000036 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [middle_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasL_Right (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasL_Right + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/plan.json new file mode 100644 index 000000000000..1f23a0e1ab62 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/plan.json @@ -0,0 +1,318 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE LEFT_TABLE (L_ID BIGINT PRIMARY KEY, FOREIGN_KEY BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='left_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "LEFT_TABLE", + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE MIDDLE_TABLE (M_ID BIGINT PRIMARY KEY, F2 BIGINT, OTHER STRING) WITH (FORMAT='JSON', KAFKA_TOPIC='middle_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "MIDDLE_TABLE", + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE RIGHT_TABLE (R_ID BIGINT PRIMARY KEY, F3 BIGINT) WITH (FORMAT='JSON', KAFKA_TOPIC='right_topic');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "RIGHT_TABLE", + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + } + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n LEFT_TABLE.L_ID L_ID,\n MIDDLE_TABLE.F2 F2,\n RIGHT_TABLE.F3 F3\nFROM LEFT_TABLE LEFT_TABLE\nINNER JOIN MIDDLE_TABLE MIDDLE_TABLE ON ((LEFT_TABLE.FOREIGN_KEY = MIDDLE_TABLE.M_ID))\nLEFT OUTER JOIN RIGHT_TABLE RIGHT_TABLE ON ((LEFT_TABLE.L_ID = RIGHT_TABLE.R_ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`L_ID` BIGINT KEY, `F2` BIGINT, `F3` BIGINT", + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "orReplace" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_TABLE", "MIDDLE_TABLE", "RIGHT_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "fkTableTableJoinV1", + "properties" : { + "queryContext" : "L_Join" + }, + "joinType" : "INNER", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Left" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "LEFT_TABLE_L_ID" ], + "selectExpressions" : [ "FOREIGN_KEY AS LEFT_TABLE_FOREIGN_KEY", "ROWTIME AS LEFT_TABLE_ROWTIME", "L_ID AS LEFT_TABLE_L_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasL_Right" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_L_Right/Source" + }, + "topicName" : "middle_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "forceChangelog" : true + }, + "keyColumnNames" : [ "MIDDLE_TABLE_M_ID" ], + "selectExpressions" : [ "F2 AS MIDDLE_TABLE_F2", "OTHER AS MIDDLE_TABLE_OTHER", "ROWTIME AS MIDDLE_TABLE_ROWTIME", "M_ID AS MIDDLE_TABLE_M_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "leftJoinExpression" : "LEFT_TABLE_FOREIGN_KEY" + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "sourceSchema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "forceChangelog" : true + }, + "keyColumnNames" : [ "RIGHT_TABLE_R_ID" ], + "selectExpressions" : [ "F3 AS RIGHT_TABLE_F3", "ROWTIME AS RIGHT_TABLE_ROWTIME", "R_ID AS RIGHT_TABLE_R_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "keyColName" : "LEFT_TABLE_L_ID" + }, + "keyColumnNames" : [ "L_ID" ], + "selectExpressions" : [ "MIDDLE_TABLE_F2 AS F2", "RIGHT_TABLE_F3 AS F3" ], + "internalFormats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + } + }, + "formats" : { + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : { + "format" : "JSON" + }, + "keyFeatures" : [ "UNWRAP_SINGLES" ] + }, + "topicName" : "OUTPUT" + }, + "queryId" : "CTAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "metric.reporters" : "", + "ksql.transient.prefix" : "transient_", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.query.error.max.queue.size" : "10", + "ksql.variable.substitution.enable" : "true", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.query.pull.metrics.enabled" : "true", + "ksql.create.or.replace.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.lambdas.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.suppress.enabled" : "false", + "ksql.query.push.scalable.enabled" : "false", + "ksql.query.push.scalable.interpreter.enabled" : "true", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.schema.registry.url" : "", + "ksql.properties.overrides.denylist" : "", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "false", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.streams.num.stream.threads" : "4", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.metrics.tags.custom" : "", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.udf.collect.metrics" : "false", + "ksql.query.pull.thread.pool.size" : "100", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/spec.json new file mode 100644 index 000000000000..47c0af8ef399 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/spec.json @@ -0,0 +1,322 @@ +{ + "version" : "7.1.0", + "timestamp" : 1627438336048, + "path" : "query-validation-tests/fk-n-way-join.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasL_Left" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasL_Right" : { + "schema" : "`MIDDLE_TABLE_M_ID` BIGINT KEY, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `MIDDLE_TABLE_ROWTIME` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Right.Source" : { + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_L_Left.Source" : { + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`L_ID` BIGINT KEY, `F2` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L_ID` BIGINT KEY, `F2` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`RIGHT_TABLE_R_ID` BIGINT KEY, `RIGHT_TABLE_F3` BIGINT, `RIGHT_TABLE_ROWTIME` BIGINT, `RIGHT_TABLE_R_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "CTAS_OUTPUT_0.L_Join" : { + "schema" : "`LEFT_TABLE_L_ID` BIGINT KEY, `LEFT_TABLE_FOREIGN_KEY` BIGINT, `LEFT_TABLE_ROWTIME` BIGINT, `LEFT_TABLE_L_ID` BIGINT, `MIDDLE_TABLE_F2` BIGINT, `MIDDLE_TABLE_OTHER` STRING, `MIDDLE_TABLE_ROWTIME` BIGINT, `MIDDLE_TABLE_M_ID` BIGINT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "Should allow fk join at start of n-way join - without fk join expressions in projection", + "inputs" : [ { + "topic" : "middle_topic", + "key" : 0, + "value" : { + "F2" : 100, + "OTHER" : "unused" + }, + "timestamp" : 0 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "FOREIGN_KEY" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "middle_topic", + "key" : 8, + "value" : { + "F2" : 10, + "OTHER" : "unused" + }, + "timestamp" : 13000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "FOREIGN_KEY" : 8 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "F2" : 100, + "F3" : null + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "F2" : 100, + "F3" : 4 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : { + "F2" : 10, + "F3" : 4 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 1, + "value" : null, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "middle_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, foreign_key BIGINT) WITH (kafka_topic='left_topic', format='JSON');", "CREATE TABLE middle_table (m_id BIGINT PRIMARY KEY, f2 BIGINT, other STRING) WITH (kafka_topic='middle_topic', format='JSON');", "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f3 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", "CREATE TABLE output AS SELECT l_id, f2, f3 FROM left_table JOIN middle_table ON foreign_key = m_id LEFT JOIN right_table ON l_id = r_id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_TABLE", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `FOREIGN_KEY` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "MIDDLE_TABLE", + "type" : "TABLE", + "schema" : "`M_ID` BIGINT KEY, `F2` BIGINT, `OTHER` STRING", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`L_ID` BIGINT KEY, `F2` BIGINT, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + }, { + "name" : "RIGHT_TABLE", + "type" : "TABLE", + "schema" : "`R_ID` BIGINT KEY, `F3` BIGINT", + "keyFormat" : { + "format" : "JSON" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ "UNWRAP_SINGLES" ], + "valueFeatures" : [ ] + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_L_Left-Reduce-changelog", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + } + }, { + "name" : "middle_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "JSON", + "features" : [ "UNWRAP_SINGLES" ] + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/topology new file mode 100644 index 000000000000..629223791785 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/fk-n-way-join_-_Should_allow_fk_join_at_start_of_n-way_join_-_without_fk_join_expressions_in_projection/7.1.0_1627438336048/topology @@ -0,0 +1,90 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000027 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000028 + Processor: KTABLE-SOURCE-0000000028 (stores: []) + --> KTABLE-MAPVALUES-0000000029 + <-- KSTREAM-SOURCE-0000000027 + Processor: KTABLE-MAPVALUES-0000000029 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000030 + <-- KTABLE-SOURCE-0000000028 + Source: KTABLE-SOURCE-0000000022 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-FK-JOIN-OUTPUT-0000000024 + <-- KTABLE-SOURCE-0000000022 + Processor: KTABLE-TRANSFORMVALUES-0000000030 (stores: []) + --> PrependAliasRight + <-- KTABLE-MAPVALUES-0000000029 + Processor: KTABLE-FK-JOIN-OUTPUT-0000000024 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-JOINTHIS-0000000033 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-0000000023 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-MAPVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000034 + <-- KTABLE-TRANSFORMVALUES-0000000030 + Processor: KTABLE-JOINOTHER-0000000034 (stores: [KTABLE-FK-JOIN-OUTPUT-STATE-STORE-0000000025]) + --> KTABLE-MERGE-0000000032 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000033 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000032 + <-- KTABLE-FK-JOIN-OUTPUT-0000000024 + Processor: KTABLE-MAPVALUES-0000000003 (stores: [KafkaTopic_L_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-MERGE-0000000032 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000035 + <-- KTABLE-JOINTHIS-0000000033, KTABLE-JOINOTHER-0000000034 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasL_Left + <-- KTABLE-MAPVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000035 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000036 + <-- KTABLE-MERGE-0000000032 + Processor: PrependAliasL_Left (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 (stores: []) + --> KTABLE-SINK-0000000014 + <-- PrependAliasL_Left + Processor: KTABLE-TOSTREAM-0000000036 (stores: []) + --> KSTREAM-SINK-0000000037 + <-- KTABLE-TRANSFORMVALUES-0000000035 + Sink: KSTREAM-SINK-0000000037 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000036 + Sink: KTABLE-SINK-0000000014 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000013 + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000007 (topics: [middle_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-MAPVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-MAPVALUES-0000000009 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Source: KTABLE-SOURCE-0000000015 (topics: [KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000012-topic]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasL_Right + <-- KTABLE-MAPVALUES-0000000009 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 + <-- KTABLE-SOURCE-0000000015 + Processor: PrependAliasL_Right (stores: []) + --> KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018 (stores: [KafkaTopic_L_Right-Reduce]) + --> KTABLE-SINK-0000000021 + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000017 + Processor: KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 (stores: [KTABLE-FK-JOIN-SUBSCRIPTION-STATE-STORE-0000000016]) + --> KTABLE-SINK-0000000021 + <-- PrependAliasL_Right + Sink: KTABLE-SINK-0000000021 (topic: KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000020-topic) + <-- KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000018, KTABLE-FK-JOIN-SUBSCRIPTION-PROCESSOR-0000000019 + diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/fk-join.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/fk-join.json index bdce0aadd6b4..5a305e4aee3f 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/fk-join.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/fk-join.json @@ -1,27 +1,27 @@ { "tests": [ { - "name": "Should fail for non-column expression in left join expression", + "name": "Should fail if types in join condition don't match", "statements": [ - "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, foreign_key BIGINT) WITH (kafka_topic='left_topic', format='JSON');", - "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f2 BIGINT) WITH (kafka_topic='right_topic', format='JSON');", - "CREATE TABLE output AS SELECT * FROM left_table JOIN right_table ON foreign_key + 1 = r_id;" + "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, foreign_key_wrong_type VARCHAR) WITH (kafka_topic='left_topic', value_format='JSON');", + "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", + "CREATE TABLE output AS SELECT * FROM left_table JOIN right_table ON foreign_key_wrong_type = r_id;" ], "expectedException": { "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Invalid join condition: foreign-key table-table joins with expressions are not supported yet. Got (LEFT_TABLE.FOREIGN_KEY + 1) = RIGHT_TABLE.R_ID." + "message": "Invalid join condition: types don't match. Got LEFT_TABLE.FOREIGN_KEY_WRONG_TYPE{STRING} = RIGHT_TABLE.R_ID{BIGINT}." } }, { - "name": "Should fail if types in join condition don't match", + "name": "Should fail if types in join condition don't match (expression)", "statements": [ - "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, foreign_key_wrong_type VARCHAR) WITH (kafka_topic='left_topic', value_format='JSON');", - "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", - "CREATE TABLE output AS SELECT * FROM left_table JOIN right_table ON foreign_key_wrong_type = r_id;" + "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", + "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", + "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table JOIN right_table ON CAST(foreign_key AS STRING) = r_id;" ], "expectedException": { "type": "io.confluent.ksql.util.KsqlStatementException", - "message": "Invalid join condition: types don't match. Got LEFT_TABLE.FOREIGN_KEY_WRONG_TYPE{STRING} = RIGHT_TABLE.R_ID{BIGINT}." + "message": "Invalid join condition: types don't match. Got CAST(LEFT_TABLE.FOREIGN_KEY AS STRING){STRING} = RIGHT_TABLE.R_ID{BIGINT}." } }, { @@ -282,6 +282,96 @@ ] } }, + { + "name": "Should support non-column reference in left join expression", + "statements": [ + "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", + "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", + "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table JOIN right_table ON foreign_key + 1 = r_id;" + ], + "inputs": [ + {"topic": "right_topic", "key": 0, "value": {"F1": "blah", "F2": 4}, "timestamp": 0}, + {"topic": "left_topic", "key": 1, "value": {"NAME": "zero", "FOREIGN_KEY": -1}, "timestamp": 10000}, + {"topic": "left_topic", "key": 0, "value": {"NAME": "foo", "FOREIGN_KEY": 99}, "timestamp": 13000}, + {"topic": "right_topic", "key": 0, "value": {"F1": "a", "F2": 10}, "timestamp": 15000}, + {"topic": "left_topic", "key": 10, "value": {"NAME": "bar", "FOREIGN_KEY": -1}, "timestamp": 16000}, + {"topic": "right_topic", "key": 0, "value": null, "timestamp": 17000}, + {"topic": "left_topic", "key": 1, "value": null, "timestamp": 18000} + ], + "outputs": [ + {"topic": "OUTPUT", "key": 1, "value": {"R_ID": 0, "NAME": "zero", "F1": "blah"}, "timestamp": 10000}, + {"topic": "OUTPUT", "key": 1, "value": {"R_ID": 0, "NAME": "zero", "F1": "a"}, "timestamp": 15000}, + {"topic": "OUTPUT", "key": 10, "value": {"R_ID": 0, "NAME": "bar", "F1": "a"}, "timestamp": 16000}, + {"topic": "OUTPUT", "key": 1, "value": null, "timestamp": 17000}, + {"topic": "OUTPUT", "key": 10, "value": null, "timestamp": 17000}, + {"topic": "OUTPUT", "key": 1, "value": null, "timestamp": 18000} + ], + "post": { + "sources": [ + {"name": "OUTPUT", "type": "table", "schema": "L_ID BIGINT KEY, R_ID BIGINT, NAME STRING, F1 STRING"} + ] + } + }, + { + "name": "Should support non-column reference in left join expression with qualifier", + "statements": [ + "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", + "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", + "CREATE TABLE output AS SELECT l_id, r_id, name, f1 FROM left_table JOIN right_table ON left_table.foreign_key + 1 = r_id;" + ], + "inputs": [ + {"topic": "right_topic", "key": 0, "value": {"F1": "blah", "F2": 4}, "timestamp": 0}, + {"topic": "left_topic", "key": 1, "value": {"NAME": "zero", "FOREIGN_KEY": -1}, "timestamp": 10000}, + {"topic": "left_topic", "key": 0, "value": {"NAME": "foo", "FOREIGN_KEY": 99}, "timestamp": 13000}, + {"topic": "right_topic", "key": 0, "value": {"F1": "a", "F2": 10}, "timestamp": 15000}, + {"topic": "left_topic", "key": 10, "value": {"NAME": "bar", "FOREIGN_KEY": -1}, "timestamp": 16000}, + {"topic": "right_topic", "key": 0, "value": null, "timestamp": 17000}, + {"topic": "left_topic", "key": 1, "value": null, "timestamp": 18000} + ], + "outputs": [ + {"topic": "OUTPUT", "key": 1, "value": {"R_ID": 0, "NAME": "zero", "F1": "blah"}, "timestamp": 10000}, + {"topic": "OUTPUT", "key": 1, "value": {"R_ID": 0, "NAME": "zero", "F1": "a"}, "timestamp": 15000}, + {"topic": "OUTPUT", "key": 10, "value": {"R_ID": 0, "NAME": "bar", "F1": "a"}, "timestamp": 16000}, + {"topic": "OUTPUT", "key": 1, "value": null, "timestamp": 17000}, + {"topic": "OUTPUT", "key": 10, "value": null, "timestamp": 17000}, + {"topic": "OUTPUT", "key": 1, "value": null, "timestamp": 18000} + ], + "post": { + "sources": [ + {"name": "OUTPUT", "type": "table", "schema": "L_ID BIGINT KEY, R_ID BIGINT, NAME STRING, F1 STRING"} + ] + } + }, + { + "name": "Should support non-column reference in left join expression with alias", + "statements": [ + "CREATE TABLE left_table (l_id BIGINT PRIMARY KEY, name VARCHAR, foreign_key BIGINT) WITH (kafka_topic='left_topic', value_format='JSON');", + "CREATE TABLE right_table (r_id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='JSON');", + "CREATE TABLE output AS SELECT lt.l_id, r_id, lt.name, f1 FROM left_table AS lt JOIN right_table ON lt.foreign_key + 1 = r_id;" + ], + "inputs": [ + {"topic": "right_topic", "key": 0, "value": {"F1": "blah", "F2": 4}, "timestamp": 0}, + {"topic": "left_topic", "key": 1, "value": {"NAME": "zero", "FOREIGN_KEY": -1}, "timestamp": 10000}, + {"topic": "left_topic", "key": 0, "value": {"NAME": "foo", "FOREIGN_KEY": 99}, "timestamp": 13000}, + {"topic": "right_topic", "key": 0, "value": {"F1": "a", "F2": 10}, "timestamp": 15000}, + {"topic": "left_topic", "key": 10, "value": {"NAME": "bar", "FOREIGN_KEY": -1}, "timestamp": 16000}, + {"topic": "right_topic", "key": 0, "value": null, "timestamp": 17000}, + {"topic": "left_topic", "key": 1, "value": null, "timestamp": 18000} + ], + "outputs": [ + {"topic": "OUTPUT", "key": 1, "value": {"R_ID": 0, "NAME": "zero", "F1": "blah"}, "timestamp": 10000}, + {"topic": "OUTPUT", "key": 1, "value": {"R_ID": 0, "NAME": "zero", "F1": "a"}, "timestamp": 15000}, + {"topic": "OUTPUT", "key": 10, "value": {"R_ID": 0, "NAME": "bar", "F1": "a"}, "timestamp": 16000}, + {"topic": "OUTPUT", "key": 1, "value": null, "timestamp": 17000}, + {"topic": "OUTPUT", "key": 10, "value": null, "timestamp": 17000}, + {"topic": "OUTPUT", "key": 1, "value": null, "timestamp": 18000} + ], + "post": { + "sources": [ + {"name": "OUTPUT", "type": "table", "schema": "L_ID BIGINT KEY, R_ID BIGINT, NAME STRING, F1 STRING"} + ] + } + }, { "name": "Should support flipped join condition", "statements": [ diff --git a/ksqldb-rest-app/src/test/resources/ksql-plan-schema/schema.json b/ksqldb-rest-app/src/test/resources/ksql-plan-schema/schema.json index da83d870f25c..486bd0ec2257 100644 --- a/ksqldb-rest-app/src/test/resources/ksql-plan-schema/schema.json +++ b/ksqldb-rest-app/src/test/resources/ksql-plan-schema/schema.json @@ -1074,10 +1074,13 @@ }, "rightSource" : { "$ref" : "#/definitions/ExecutionStep" + }, + "leftJoinExpression" : { + "type" : "string" } }, "title" : "fkTableTableJoinV1", - "required" : [ "@type", "properties", "joinType", "leftJoinColumnName", "formats", "leftSource", "rightSource" ] + "required" : [ "@type", "properties", "joinType", "formats", "leftSource", "rightSource" ] }, "RefinementInfo" : { "type" : "object", diff --git a/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ExecutionStepFactory.java b/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ExecutionStepFactory.java index c04cb0f7dc51..d645f64d26e9 100644 --- a/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ExecutionStepFactory.java +++ b/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ExecutionStepFactory.java @@ -334,10 +334,11 @@ public static TableTableJoin tableTableJoin( foreignKeyTableTableJoin( final QueryContext.Stacker stacker, final JoinType joinType, - final ColumnName leftJoinColumnName, + final Optional leftJoinColumnName, final Formats formats, final ExecutionStep> left, - final ExecutionStep> right + final ExecutionStep> right, + final Optional leftJoinExpression ) { final QueryContext queryContext = stacker.getQueryContext(); return new ForeignKeyTableTableJoin<>( @@ -346,7 +347,8 @@ public static TableTableJoin tableTableJoin( leftJoinColumnName, formats, left, - right + right, + leftJoinExpression ); } diff --git a/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ForeignKeyJoinParamsFactory.java b/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ForeignKeyJoinParamsFactory.java index b01488ba5dfe..11db5395708e 100644 --- a/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ForeignKeyJoinParamsFactory.java +++ b/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ForeignKeyJoinParamsFactory.java @@ -15,11 +15,10 @@ package io.confluent.ksql.execution.streams; -import io.confluent.ksql.name.ColumnName; -import io.confluent.ksql.schema.ksql.Column; +import io.confluent.ksql.execution.transform.ExpressionEvaluator; +import io.confluent.ksql.logging.processing.ProcessingLogger; import io.confluent.ksql.schema.ksql.LogicalSchema; import io.confluent.ksql.schema.ksql.LogicalSchema.Builder; -import java.util.Optional; public final class ForeignKeyJoinParamsFactory { @@ -27,15 +26,16 @@ private ForeignKeyJoinParamsFactory() { } public static ForeignKeyJoinParams create( - final ColumnName leftJoinColumnName, + final ExpressionEvaluator expressionEvaluator, final LogicalSchema leftSchema, - final LogicalSchema rightSchema + final LogicalSchema rightSchema, + final ProcessingLogger processingLogger ) { if (rightSchema.key().size() != 1) { throw new IllegalStateException("rightSchema must have single column key"); } return new ForeignKeyJoinParams<>( - createKeyExtractor(leftSchema, leftJoinColumnName), + new KsqlKeyExtractor<>(expressionEvaluator, processingLogger), new KsqlValueJoiner(leftSchema.value().size(), rightSchema.value().size(), 0), createSchema(leftSchema, rightSchema) ); @@ -52,16 +52,4 @@ public static LogicalSchema createSchema( return builder.build(); } - - private static KsqlKeyExtractor createKeyExtractor( - final LogicalSchema leftSchema, - final ColumnName leftJoinColumnName) { - - final Optional leftJoinColumn = leftSchema.findValueColumn(leftJoinColumnName); - if (!leftJoinColumn.isPresent()) { - throw new IllegalStateException("Could not find join column in left input table."); - } - - return new KsqlKeyExtractor<>(leftJoinColumn.get().index()); - } } diff --git a/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ForeignKeyTableTableJoinBuilder.java b/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ForeignKeyTableTableJoinBuilder.java index 4565392f9a1d..c821efa61b16 100644 --- a/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ForeignKeyTableTableJoinBuilder.java +++ b/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/ForeignKeyTableTableJoinBuilder.java @@ -16,12 +16,19 @@ package io.confluent.ksql.execution.streams; import io.confluent.ksql.GenericRow; +import io.confluent.ksql.execution.codegen.CodeGenRunner; +import io.confluent.ksql.execution.expression.tree.Expression; +import io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp; import io.confluent.ksql.execution.plan.ForeignKeyTableTableJoin; import io.confluent.ksql.execution.plan.Formats; import io.confluent.ksql.execution.plan.KTableHolder; import io.confluent.ksql.execution.runtime.RuntimeBuildContext; +import io.confluent.ksql.execution.transform.ExpressionEvaluator; +import io.confluent.ksql.logging.processing.ProcessingLogger; +import io.confluent.ksql.name.ColumnName; import io.confluent.ksql.schema.ksql.LogicalSchema; import io.confluent.ksql.schema.ksql.PhysicalSchema; +import java.util.Optional; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.Materialized; @@ -40,8 +47,35 @@ public static KTableHolder build( final LogicalSchema leftSchema = left.getSchema(); final LogicalSchema rightSchema = right.getSchema(); + final ProcessingLogger logger = buildContext.getProcessingLogger( + join.getProperties().getQueryContext() + ); + + final ExpressionEvaluator expressionEvaluator; + final CodeGenRunner codeGenRunner = new CodeGenRunner( + leftSchema, + buildContext.getKsqlConfig(), + buildContext.getFunctionRegistry() + ); + + final Optional leftColumnName = join.getLeftJoinColumnName(); + final Optional leftJoinExpression = join.getLeftJoinExpression(); + if (leftColumnName.isPresent()) { + expressionEvaluator = codeGenRunner.buildCodeGenFromParseTree( + new UnqualifiedColumnReferenceExp(leftColumnName.get()), + "Left Join Expression" + ); + } else if (leftJoinExpression.isPresent()) { + expressionEvaluator = codeGenRunner.buildCodeGenFromParseTree( + leftJoinExpression.get(), + "Left Join Expression" + ); + } else { + throw new IllegalStateException("Both leftColumnName and leftJoinExpression are empty."); + } + final ForeignKeyJoinParams joinParams = ForeignKeyJoinParamsFactory - .create(join.getLeftJoinColumnName(), leftSchema, rightSchema); + .create(expressionEvaluator, leftSchema, rightSchema, logger); final Formats formats = join.getFormats(); diff --git a/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/KsqlKeyExtractor.java b/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/KsqlKeyExtractor.java index 8302ab0e666b..0d017acc55f0 100644 --- a/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/KsqlKeyExtractor.java +++ b/ksqldb-streams/src/main/java/io/confluent/ksql/execution/streams/KsqlKeyExtractor.java @@ -15,30 +15,43 @@ package io.confluent.ksql.execution.streams; -import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; import io.confluent.ksql.GenericKey; import io.confluent.ksql.GenericRow; +import io.confluent.ksql.execution.transform.ExpressionEvaluator; +import io.confluent.ksql.logging.processing.ProcessingLogger; import java.util.Objects; import java.util.function.Function; +import java.util.function.Supplier; public final class KsqlKeyExtractor implements Function { - private final int leftJoinColumnIndex; + private final ExpressionEvaluator expressionEvaluator; + private final ProcessingLogger processingLogger; + private final Supplier errorMessage; - KsqlKeyExtractor(final int leftJoinColumnIndex) { - checkArgument( - leftJoinColumnIndex >= 0, - "leftJoinColumnIndex negative: " + leftJoinColumnIndex + KsqlKeyExtractor(final ExpressionEvaluator expressionEvaluator, + final ProcessingLogger processingLogger) { + this.expressionEvaluator = requireNonNull(expressionEvaluator); + this.processingLogger = processingLogger; + errorMessage = () -> String.format( + "Error calculating left join expression: `%s`.", + expressionEvaluator.getExpression() ); - - this.leftJoinColumnIndex = leftJoinColumnIndex; } @SuppressWarnings("unchecked") @Override public KRightT apply(final GenericRow left) { - return (KRightT) GenericKey.genericKey(left.get(leftJoinColumnIndex)); + return (KRightT) GenericKey.genericKey( + expressionEvaluator.evaluate( + left, + null, + processingLogger, + errorMessage + ) + ); } @Override @@ -50,18 +63,22 @@ public boolean equals(final Object o) { return false; } final KsqlKeyExtractor that = (KsqlKeyExtractor) o; - return leftJoinColumnIndex == that.leftJoinColumnIndex; + return Objects.equals(expressionEvaluator, that.expressionEvaluator) + && Objects.equals(processingLogger, that.processingLogger) + && Objects.equals(errorMessage, that.errorMessage); } @Override public int hashCode() { - return Objects.hash(leftJoinColumnIndex); + return Objects.hash(expressionEvaluator, processingLogger, errorMessage); } @Override public String toString() { return "KsqlKeyExtractor{" - + "leftJoinColumnIndex=" + leftJoinColumnIndex + + "expressionEvaluator=" + expressionEvaluator + + "processingLogger=" + processingLogger + + "errorMessage=" + errorMessage + '}'; } } diff --git a/ksqldb-streams/src/test/java/io/confluent/ksql/execution/streams/ForeignKeyJoinParamsFactoryTest.java b/ksqldb-streams/src/test/java/io/confluent/ksql/execution/streams/ForeignKeyJoinParamsFactoryTest.java index d4b4b1c1ab9b..ee9d942ce636 100644 --- a/ksqldb-streams/src/test/java/io/confluent/ksql/execution/streams/ForeignKeyJoinParamsFactoryTest.java +++ b/ksqldb-streams/src/test/java/io/confluent/ksql/execution/streams/ForeignKeyJoinParamsFactoryTest.java @@ -19,10 +19,13 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; +import io.confluent.ksql.execution.transform.ExpressionEvaluator; import io.confluent.ksql.name.ColumnName; import io.confluent.ksql.schema.ksql.LogicalSchema; import io.confluent.ksql.schema.ksql.types.SqlTypes; +import java.util.Optional; import org.junit.Test; public class ForeignKeyJoinParamsFactoryTest { @@ -43,12 +46,9 @@ public class ForeignKeyJoinParamsFactoryTest { @Test public void shouldBuildCorrectKeyedSchema() { - // Given: - final ColumnName leftJoinColumnName = ColumnName.of("L_FOREIGN_KEY"); - // When: final ForeignKeyJoinParams joinParams = - ForeignKeyJoinParamsFactory.create(leftJoinColumnName, LEFT_SCHEMA, RIGHT_SCHEMA); + ForeignKeyJoinParamsFactory.create(mock(ExpressionEvaluator.class), LEFT_SCHEMA, RIGHT_SCHEMA, null); // Then: assertThat(joinParams.getSchema(), is(LogicalSchema.builder() @@ -62,21 +62,4 @@ public void shouldBuildCorrectKeyedSchema() { .build()) ); } - - @Test - public void shouldThrowIfJoinColumnNotFound() { - // Given: - final ColumnName leftJoinColumnName = ColumnName.of("L_UNKNOWN"); - - final Exception e = assertThrows( - IllegalStateException.class, - () -> ForeignKeyJoinParamsFactory.create(leftJoinColumnName, LEFT_SCHEMA, RIGHT_SCHEMA) - ); - - // Then: - assertThat( - e.getMessage(), - containsString("Could not find join column in left input table.") - ); - } } \ No newline at end of file diff --git a/ksqldb-streams/src/test/java/io/confluent/ksql/execution/streams/ForeignKeyTableTableJoinBuilderTest.java b/ksqldb-streams/src/test/java/io/confluent/ksql/execution/streams/ForeignKeyTableTableJoinBuilderTest.java index 7dc53ea68a63..ffbcad585da0 100644 --- a/ksqldb-streams/src/test/java/io/confluent/ksql/execution/streams/ForeignKeyTableTableJoinBuilderTest.java +++ b/ksqldb-streams/src/test/java/io/confluent/ksql/execution/streams/ForeignKeyTableTableJoinBuilderTest.java @@ -38,10 +38,13 @@ import io.confluent.ksql.execution.plan.PlanBuilder; import io.confluent.ksql.execution.plan.PlanInfo; import io.confluent.ksql.execution.runtime.RuntimeBuildContext; +import io.confluent.ksql.function.FunctionRegistry; import io.confluent.ksql.name.ColumnName; import io.confluent.ksql.schema.ksql.LogicalSchema; import io.confluent.ksql.schema.ksql.types.SqlTypes; import io.confluent.ksql.serde.SerdeFeatures; +import io.confluent.ksql.util.KsqlConfig; +import java.util.Optional; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.Materialized; @@ -129,8 +132,12 @@ public void init() { when(formats.getKeyFeatures()).thenReturn(mock(SerdeFeatures.class)); when(formats.getValueFeatures()).thenReturn(mock(SerdeFeatures.class)); + final RuntimeBuildContext context = mock(RuntimeBuildContext.class); + when((context.getFunctionRegistry())).thenReturn(mock(FunctionRegistry.class)); + when((context.getKsqlConfig())).thenReturn(mock(KsqlConfig.class)); + planBuilder = new KSPlanBuilder( - mock(RuntimeBuildContext.class), + context, mock(SqlPredicateFactory.class), mock(AggregateParamsFactory.class), mock(StreamsFactories.class) @@ -149,7 +156,7 @@ public void shouldDoLeftJoinOnNonKey() { // Then: verify(leftKTable).leftJoin( same(rightKTable), - eq(new KsqlKeyExtractor<>(1)), + any(KsqlKeyExtractor.class), eq(new KsqlValueJoiner(LEFT_SCHEMA.value().size(), RIGHT_SCHEMA.value().size(), 0)), any(Materialized.class) ); @@ -178,7 +185,7 @@ public void shouldDoLeftJoinOnKey() { // Then: verify(leftKTable).leftJoin( same(rightKTable), - eq(new KsqlKeyExtractor<>(2)), + any(KsqlKeyExtractor.class), eq(new KsqlValueJoiner(LEFT_SCHEMA.value().size(), RIGHT_SCHEMA.value().size(), 0)), any(Materialized.class) ); @@ -199,7 +206,7 @@ public void shouldDoLeftJoinOnSubKey() { // Then: verify(leftKTableMultiKey).leftJoin( same(rightKTable), - eq(new KsqlKeyExtractor<>(3)), + any(KsqlKeyExtractor.class), eq(new KsqlValueJoiner(LEFT_SCHEMA_MULTI_KEY.value().size(), RIGHT_SCHEMA.value().size(), 0)), any(Materialized.class) ); @@ -220,7 +227,7 @@ public void shouldDoInnerJoinOnNonKey() { // Then: verify(leftKTable).join( same(rightKTable), - eq(new KsqlKeyExtractor<>(1)), + any(KsqlKeyExtractor.class), eq(new KsqlValueJoiner(LEFT_SCHEMA.value().size(), RIGHT_SCHEMA.value().size(), 0)), any(Materialized.class) ); @@ -249,7 +256,7 @@ public void shouldDoInnerJoinOnKey() { // Then: verify(leftKTable).join( same(rightKTable), - eq(new KsqlKeyExtractor<>(2)), + any(KsqlKeyExtractor.class), eq(new KsqlValueJoiner(LEFT_SCHEMA.value().size(), RIGHT_SCHEMA.value().size(), 0)), any(Materialized.class) ); @@ -270,7 +277,7 @@ public void shouldDoInnerJoinOnSubKey() { // Then: verify(leftKTableMultiKey).join( same(rightKTable), - eq(new KsqlKeyExtractor<>(3)), + any(KsqlKeyExtractor.class), eq(new KsqlValueJoiner(LEFT_SCHEMA_MULTI_KEY.value().size(), RIGHT_SCHEMA.value().size(), 0)), any(Materialized.class) ); @@ -323,10 +330,11 @@ private void givenLeftJoin(final ExecutionStep> left, join = new ForeignKeyTableTableJoin<>( new ExecutionStepPropertiesV1(ctx), JoinType.LEFT, - leftJoinColumnName, + Optional.of(leftJoinColumnName), formats, left, - right + right, + Optional.empty() ); } @@ -335,10 +343,11 @@ private void givenInnerJoin(final ExecutionStep> left, join = new ForeignKeyTableTableJoin<>( new ExecutionStepPropertiesV1(ctx), JoinType.INNER, - leftJoinColumnName, + Optional.of(leftJoinColumnName), formats, left, - right + right, + Optional.empty() ); } }